<!DOCTYPE html>
<html xmlns="http://www.w3.org/1999/xhtml">
  <head>

    <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
    <meta content="Cask Data, Inc." name="author" />
<meta content="Release notes for the Cask Data Application Platform" name="description" />
<meta content="Copyright © 2014-2018 Cask Data, Inc." name="copyright" />


    <meta name="git_release" content="6.1.1">
    <meta name="git_hash" content="05fbac36f9f7aadeb44f5728cea35136dbc243e5">
    <meta name="git_timestamp" content="2020-02-09 08:22:47 +0800">
    <title>Cask Data Application Platform 发行说明</title>

    <link rel="stylesheet" href="_static/cdap-bootstrap.css" type="text/css" />
    <link rel="stylesheet" href="_static/pygments.css" type="text/css" />
    <link rel="stylesheet" href="_static/bootstrap-3.3.6/css/bootstrap.min.css" type="text/css" />
    <link rel="stylesheet" href="_static/bootstrap-3.3.6/css/bootstrap-theme.min.css" type="text/css" />
    <link rel="stylesheet" href="_static/css/bootstrap-sphinx.css" type="text/css" />
    <link rel="stylesheet" href="_static/css/cdap-dynamicscrollspy-4.css" type="text/css" />
    <link rel="stylesheet" href="_static/css/jquery.mCustomScrollbar.css" type="text/css" />
    <link rel="stylesheet" href="_static/css/cdap-jquery.mCustomScrollbar.css" type="text/css" />
    <link rel="stylesheet" href="_static/css/abixTreeList-2.css" type="text/css" />
    <link rel="stylesheet" href="_static/cdap-bootstrap.css" type="text/css" />

    <script type="text/javascript">
      var DOCUMENTATION_OPTIONS = {
        URL_ROOT:    '',
        VERSION:     '6.1.1',
        COLLAPSE_INDEX: false,
        FILE_SUFFIX: '.html',
        HAS_SOURCE:  false
      };
    </script>
    <script type="text/javascript" src="_static/jquery.js"></script>
    <script type="text/javascript" src="_static/underscore.js"></script>
    <script type="text/javascript" src="_static/doctools.js"></script>
    <script type="text/javascript" src="_static/language_data.js"></script>

    <link rel="shortcut icon" href="_static/favicon.ico"/>
    <link rel="index" title="Index" href="genindex.html" />
    <link rel="search" title="Search" href="search.html" />
    <link rel="top" title="Cask Data Application Platform 6.1.1 Documentation" href="index.html" />
    <link rel="next" title="Licenses, and Dependencies" href="licenses/index.html" />
    <link rel="prev" title="Javadocs" href="javadocs/index.html" />
    <!-- block extrahead -->
    <meta charset='utf-8'>
    <meta http-equiv='X-UA-Compatible' content='IE=edge,chrome=1'>
    <meta name='viewport' content='width=device-width, initial-scale=1.0, maximum-scale=1'>
    <meta name="apple-mobile-web-app-capable" content="yes">
    <!-- block extrahead end -->

</head>
<body role="document">

<!-- block navbar -->
<div id="navbar" class="navbar navbar-inverse navbar-default navbar-fixed-top">
    <div class="container-fluid">
      <div class="row">
        <div class="navbar-header">
          <!-- .btn-navbar is used as the toggle for collapsed navbar content -->
          <a class="navbar-brand" href="table-of-contents/../../index.html">
            <span><img alt="CDAP logo" src="_static/cdap_logo.svg"/></span>
          </a>

          <button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".nav-collapse">
            <span class="icon-bar"></span>
            <span class="icon-bar"></span>
            <span class="icon-bar"></span>
          </button>

          <div class="pull-right">
            <div class="dropdown version-dropdown">
              <a href="#" class="dropdown-toggle" data-toggle="dropdown"
                role="button" aria-haspopup="true" aria-expanded="false">
                v 6.1.1 <span class="caret"></span>
              </a>
              <ul class="dropdown-menu">
                <li><a href="//docs.cdap.io/cdap/5.1.2/en/index.html">v 5.1.2</a></li>
                <li><a href="//docs.cdap.io/cdap/4.3.4/en/index.html">v 4.3.4</a></li>
              </ul>
            </div>
          </div>
          <form class="navbar-form navbar-right navbar-search" action="search.html" method="get">
            <div class="form-group">
              <div class="navbar-search-image material-icons"></div>
              <input type="text" name="q" class="form-control" placeholder="  Search" />
            </div>
            <input type="hidden" name="check_keywords" value="yes" />
            <input type="hidden" name="area" value="default" />
          </form>

          <div class="collapse navbar-collapse nav-collapse navbar-right navbar-navigation">
            <ul class="nav navbar-nav"><li class="docsite-nav-tab-container"><a class="docsite-nav-tab-link " href="table-of-contents/../../index.html">简介</a></li><li class="docsite-nav-tab-container"><a class="docsite-nav-tab-link " href="table-of-contents/../../guides.html">手册</a></li><li class="docsite-nav-tab-container"><a class="docsite-nav-tab-link current" href="table-of-contents/../../reference-manual/index.html">参考</a></li><li class="docsite-nav-tab-container"><a class="docsite-nav-tab-link " href="table-of-contents/../../faqs/index.html">帮助</a></li>
            </ul>
          </div>

        </div>
      </div>
    </div>
  </div><!-- block navbar end -->
<!-- block main content -->
<div class="main-container container">
  <div class="row"><div class="col-md-2">
      <div id="sidebar" class="bs-sidenav scrollable-y-outside" role="complementary">
<!-- theme_manual: reference-manual -->
<!-- theme_manual_highlight: reference -->
<!-- sidebar_title_link: Reference -->

  <div role="note" aria-label="manuals links"><h3>Reference</h3>
    <ul class="reference-page-menu">
      <nav class="pagenav">
      <ul class="current">
<li class="toctree-l1"><a class="reference internal" href="index.html"> Introduction</a></li>
<li class="toctree-l1"><a class="reference internal" href="characters.html"> Supported Characters</a></li>
<li class="toctree-l1"><a class="reference internal" href="cli-api.html"> Command Line Interface API</a></li>
<li class="toctree-l1"><a class="reference internal" href="http-restful-api/index.html"> HTTP RESTful API</a><ul>
<li class="toctree-l2"><a class="reference internal" href="http-restful-api/introduction.html">Introduction</a></li>
<li class="toctree-l2"><a class="reference internal" href="http-restful-api/artifact.html">Artifact</a></li>
<li class="toctree-l2"><a class="reference internal" href="http-restful-api/configuration.html">Configuration</a></li>
<li class="toctree-l2"><a class="reference internal" href="http-restful-api/dataset.html">Dataset</a></li>
<li class="toctree-l2"><a class="reference internal" href="http-restful-api/lifecycle.html">Lifecycle</a></li>
<li class="toctree-l2"><a class="reference internal" href="http-restful-api/logging.html">Logging</a></li>
<li class="toctree-l2"><a class="reference internal" href="http-restful-api/metadata.html">Metadata</a></li>
<li class="toctree-l2"><a class="reference internal" href="http-restful-api/metrics.html">Metrics</a></li>
<li class="toctree-l2"><a class="reference internal" href="http-restful-api/monitor.html">Monitor</a></li>
<li class="toctree-l2"><a class="reference internal" href="http-restful-api/namespace.html">Namespace</a></li>
<li class="toctree-l2"><a class="reference internal" href="http-restful-api/dashboard.html">Dashboard</a></li>
<li class="toctree-l2"><a class="reference internal" href="http-restful-api/preferences.html">Preferences</a></li>
<li class="toctree-l2"><a class="reference internal" href="http-restful-api/profile.html">Profile</a></li>
<li class="toctree-l2"><a class="reference internal" href="http-restful-api/query.html">Query</a></li>
<li class="toctree-l2"><a class="reference internal" href="http-restful-api/reports.html">Reports</a></li>
<li class="toctree-l2"><a class="reference internal" href="http-restful-api/security.html">Security</a></li>
<li class="toctree-l2"><a class="reference internal" href="http-restful-api/service.html">Service</a></li>
<li class="toctree-l2"><a class="reference internal" href="http-restful-api/transactions.html">Transactions</a></li>
<li class="toctree-l2"><a class="reference internal" href="http-restful-api/workflow.html">Workflow</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="java-client-api.html"> Java Client API</a></li>
<li class="toctree-l1"><a class="reference internal" href="javadocs/index.html"> Javadocs</a></li>
<li class="toctree-l1 current"><a class="current reference internal" href="#"> 发行说明</a></li>
<li class="toctree-l1"><a class="reference internal" href="licenses/index.html"> Trademarks, Licenses, and Dependencies</a></li>
<li class="toctree-l1"><a class="reference internal" href="glossary.html"> 术语表</a></li>
<li class="toctree-l1"><a class="reference internal" href="genindex.html"> Index</a></li>
</ul>
</nav>
    </ul>
  </div></div>
    </div><div class="col-md-8 content" id="main-content">
    
  <span class="target" id="overview-release-notes"></span><div class="section" id="cask-data-application-platform">
<span id="release-notes"></span><span id="index-0"></span><h1>Cask Data Application Platform 发行说明<a class="headerlink" href="#cask-data-application-platform" title="Permalink to this headline">🔗</a></h1>
<div class="contents faq local topic" id="contents">
<ul class="simple">
<li><a class="reference internal" href="#release-6-1-1" id="id150">Release 6.1.1</a><ul>
<li><a class="reference internal" href="#summary" id="id151">Summary</a></li>
<li><a class="reference internal" href="#new-features" id="id152">New Features</a></li>
<li><a class="reference internal" href="#deprecation" id="id153">Deprecation</a></li>
<li><a class="reference internal" href="#bug-fixes" id="id154">Bug Fixes</a></li>
<li><a class="reference internal" href="#improvements" id="id155">Improvements</a></li>
</ul>
</li>
<li><a class="reference internal" href="#release-6-0-0" id="id156">Release 6.0.0</a><ul>
<li><a class="reference internal" href="#id1" id="id157">Summary</a></li>
<li><a class="reference internal" href="#id2" id="id158">New Features</a></li>
<li><a class="reference internal" href="#id3" id="id159">Improvements</a></li>
<li><a class="reference internal" href="#id4" id="id160">Bug Fixes</a></li>
<li><a class="reference internal" href="#deprecated-and-removed-features" id="id161">Deprecated and Removed Features</a></li>
</ul>
</li>
<li><a class="reference internal" href="#release-5-1-2" id="id162">Release 5.1.2</a><ul>
<li><a class="reference internal" href="#id5" id="id163">Improvements</a></li>
<li><a class="reference internal" href="#id6" id="id164">Bug Fixes</a></li>
</ul>
</li>
<li><a class="reference internal" href="#release-5-1-1" id="id165">Release 5.1.1</a><ul>
<li><a class="reference internal" href="#id7" id="id166">Improvements</a></li>
<li><a class="reference internal" href="#id8" id="id167">Bug Fixes</a></li>
</ul>
</li>
<li><a class="reference internal" href="#release-5-1-0" id="id168">Release 5.1.0</a><ul>
<li><a class="reference internal" href="#id9" id="id169">Summary</a></li>
<li><a class="reference internal" href="#id10" id="id170">New Features</a></li>
<li><a class="reference internal" href="#id11" id="id171">Improvements</a></li>
<li><a class="reference internal" href="#id12" id="id172">Bug Fixes</a></li>
<li><a class="reference internal" href="#id13" id="id173">Deprecated and Removed Features</a></li>
</ul>
</li>
<li><a class="reference internal" href="#release-5-0-0" id="id174">Release 5.0.0</a><ul>
<li><a class="reference internal" href="#id14" id="id175">Summary</a></li>
<li><a class="reference internal" href="#id15" id="id176">New Features</a></li>
<li><a class="reference internal" href="#id18" id="id177">Improvements</a></li>
<li><a class="reference internal" href="#id19" id="id178">Bug Fixes</a></li>
<li><a class="reference internal" href="#id20" id="id179">Deprecated and Removed Features</a></li>
<li><a class="reference internal" href="#known-issues" id="id180">Known Issues</a></li>
</ul>
</li>
<li><a class="reference internal" href="#release-4-3-4" id="id181">Release 4.3.4</a><ul>
<li><a class="reference internal" href="#id21" id="id182">Improvements</a></li>
<li><a class="reference internal" href="#id22" id="id183">Bug Fixes</a></li>
</ul>
</li>
<li><a class="reference internal" href="#release-4-3-3" id="id184">Release 4.3.3</a><ul>
<li><a class="reference internal" href="#id23" id="id185">Improvements</a></li>
<li><a class="reference internal" href="#id24" id="id186">Bug Fixes</a></li>
</ul>
</li>
<li><a class="reference internal" href="#release-4-3-2" id="id187">Release 4.3.2</a><ul>
<li><a class="reference internal" href="#id25" id="id188">New Features</a></li>
<li><a class="reference internal" href="#id26" id="id189">Improvements</a></li>
<li><a class="reference internal" href="#id27" id="id190">Bug Fixes</a></li>
</ul>
</li>
<li><a class="reference internal" href="#release-4-3-1" id="id191">Release 4.3.1</a><ul>
<li><a class="reference internal" href="#id28" id="id192">New Features</a></li>
<li><a class="reference internal" href="#id29" id="id193">Improvements</a></li>
<li><a class="reference internal" href="#id30" id="id194">Bug Fixes</a></li>
</ul>
</li>
<li><a class="reference internal" href="#release-4-3-0" id="id195">Release 4.3.0</a><ul>
<li><a class="reference internal" href="#id31" id="id196">Summary</a></li>
<li><a class="reference internal" href="#id32" id="id197">New Features</a></li>
<li><a class="reference internal" href="#enhancements" id="id198">数据流管道 Enhancements</a></li>
<li><a class="reference internal" href="#triggers" id="id199">Triggers</a></li>
<li><a class="reference internal" href="#id33" id="id200">数据预处理 Enhancements</a></li>
<li><a class="reference internal" href="#spark-enhancements" id="id201">Spark Enhancements</a></li>
<li><a class="reference internal" href="#governance-and-security-enhancements" id="id202">Governance and Security Enhancements</a></li>
<li><a class="reference internal" href="#id34" id="id203">Other New Features</a></li>
<li><a class="reference internal" href="#id35" id="id204">Bug Fixes</a></li>
</ul>
</li>
<li><a class="reference internal" href="#release-4-2-0" id="id205">Release 4.2.0</a><ul>
<li><a class="reference internal" href="#id36" id="id206">Summary</a></li>
<li><a class="reference internal" href="#id37" id="id207">New Features</a></li>
<li><a class="reference internal" href="#id38" id="id208">Spark Enhancements</a></li>
<li><a class="reference internal" href="#enhanced" id="id209">Enhanced 数据预处理</a></li>
<li><a class="reference internal" href="#event-driven-schedules" id="id210">Event Driven Schedules</a></li>
<li><a class="reference internal" href="#id39" id="id211">Other New Features</a></li>
<li><a class="reference internal" href="#id40" id="id212">Bug fixes</a></li>
<li><a class="reference internal" href="#deprecations" id="id213">Deprecations</a></li>
</ul>
</li>
<li><a class="reference internal" href="#release-4-1-1" id="id214">Release 4.1.1</a><ul>
<li><a class="reference internal" href="#id41" id="id215">Summary</a></li>
<li><a class="reference internal" href="#id42" id="id216">New Features</a></li>
<li><a class="reference internal" href="#id45" id="id217">Improvements</a></li>
<li><a class="reference internal" href="#id47" id="id218">Bug Fixes</a></li>
<li><a class="reference internal" href="#id52" id="id219">Known Issues</a></li>
<li><a class="reference internal" href="#api-changes" id="id220">API Changes</a></li>
</ul>
</li>
<li><a class="reference internal" href="#release-4-1-0" id="id221">Release 4.1.0</a><ul>
<li><a class="reference internal" href="#id54" id="id222">New Features</a></li>
<li><a class="reference internal" href="#id57" id="id223">Improvements</a></li>
<li><a class="reference internal" href="#id58" id="id224">Bug Fixes</a></li>
<li><a class="reference internal" href="#id59" id="id225">Known Issues</a></li>
<li><a class="reference internal" href="#id60" id="id226">API Changes</a></li>
<li><a class="reference internal" href="#id61" id="id227">Deprecated and Removed Features</a></li>
</ul>
</li>
<li><a class="reference internal" href="#release-4-0-1" id="id228">Release 4.0.1</a><ul>
<li><a class="reference internal" href="#improvement" id="id229">Improvement</a></li>
<li><a class="reference internal" href="#id62" id="id230">Bug Fixes</a></li>
</ul>
</li>
<li><a class="reference internal" href="#release-4-0-0" id="id231">Release 4.0.0</a><ul>
<li><a class="reference internal" href="#id63" id="id232">New Features</a></li>
<li><a class="reference internal" href="#id64" id="id233">Improvements</a></li>
<li><a class="reference internal" href="#id65" id="id234">Bug Fixes</a></li>
<li><a class="reference internal" href="#id66" id="id235">Known Issues</a></li>
<li><a class="reference internal" href="#id67" id="id236">API Changes</a></li>
<li><a class="reference internal" href="#id68" id="id237">Deprecated and Removed Features</a></li>
</ul>
</li>
<li><a class="reference internal" href="#release-3-6-0" id="id238">Release 3.6.0</a><ul>
<li><a class="reference internal" href="#id69" id="id239">Improvements</a></li>
<li><a class="reference internal" href="#id70" id="id240">Bug Fixes</a></li>
</ul>
</li>
<li><a class="reference internal" href="#release-3-5-2" id="id241">Release 3.5.2</a><ul>
<li><a class="reference internal" href="#id71" id="id242">Known Issues</a></li>
<li><a class="reference internal" href="#id72" id="id243">Improvements</a></li>
<li><a class="reference internal" href="#id73" id="id244">Bug Fixes</a></li>
</ul>
</li>
<li><a class="reference internal" href="#release-3-5-1" id="id245">Release 3.5.1</a><ul>
<li><a class="reference internal" href="#id74" id="id246">Known Issues</a></li>
<li><a class="reference internal" href="#id75" id="id247">Improvements</a></li>
<li><a class="reference internal" href="#id76" id="id248">Bug Fixes</a></li>
</ul>
</li>
<li><a class="reference internal" href="#release-3-5-0" id="id249">Release 3.5.0</a><ul>
<li><a class="reference internal" href="#id77" id="id250">Known Issues</a></li>
<li><a class="reference internal" href="#id78" id="id251">API Changes</a></li>
<li><a class="reference internal" href="#id79" id="id252">New Features</a></li>
<li><a class="reference internal" href="#changes" id="id253">Changes</a></li>
<li><a class="reference internal" href="#id80" id="id254">Improvements</a></li>
<li><a class="reference internal" href="#id81" id="id255">Bug Fixes</a></li>
</ul>
</li>
<li><a class="reference internal" href="#release-3-4-1" id="id256">Release 3.4.1</a><ul>
<li><a class="reference internal" href="#id82" id="id257">Bug Fixes</a></li>
<li><a class="reference internal" href="#id83" id="id258">Known Issues</a></li>
</ul>
</li>
<li><a class="reference internal" href="#release-3-4-0" id="id259">Release 3.4.0</a><ul>
<li><a class="reference internal" href="#id84" id="id260">API Changes</a></li>
<li><a class="reference internal" href="#id85" id="id261">New Features</a></li>
<li><a class="reference internal" href="#id86" id="id262">Improvements</a></li>
<li><a class="reference internal" href="#id87" id="id263">Bug Fixes</a></li>
<li><a class="reference internal" href="#id88" id="id264">Deprecated and Removed Features</a></li>
<li><a class="reference internal" href="#known-issues-340" id="id265">Known Issues</a></li>
</ul>
</li>
<li><a class="reference internal" href="#release-3-3-3" id="id266">Release 3.3.3</a><ul>
<li><a class="reference internal" href="#bug-fix" id="id267">Bug Fix</a></li>
</ul>
</li>
<li><a class="reference internal" href="#release-3-3-2" id="id268">Release 3.3.2</a><ul>
<li><a class="reference internal" href="#id90" id="id269">Improvements</a></li>
<li><a class="reference internal" href="#id91" id="id270">Bug Fixes</a></li>
</ul>
</li>
<li><a class="reference internal" href="#release-3-3-1" id="id271">Release 3.3.1</a><ul>
<li><a class="reference internal" href="#id92" id="id272">Improvements</a></li>
<li><a class="reference internal" href="#id93" id="id273">Bug Fixes</a></li>
<li><a class="reference internal" href="#id94" id="id274">Deprecated and Removed Features</a></li>
<li><a class="reference internal" href="#id95" id="id275">Known Issues</a></li>
</ul>
</li>
<li><a class="reference internal" href="#release-3-3-0" id="id276">Release 3.3.0</a><ul>
<li><a class="reference internal" href="#id96" id="id277">New Features</a></li>
<li><a class="reference internal" href="#id97" id="id278">Improvements</a></li>
<li><a class="reference internal" href="#id98" id="id279">Bug Fixes</a></li>
<li><a class="reference internal" href="#id99" id="id280">Deprecated and Removed Features</a></li>
<li><a class="reference internal" href="#id100" id="id281">Known Issues</a></li>
</ul>
</li>
<li><a class="reference internal" href="#release-3-2-1" id="id282">Release 3.2.1</a><ul>
<li><a class="reference internal" href="#id101" id="id283">New Features</a></li>
<li><a class="reference internal" href="#id102" id="id284">Improvements</a></li>
<li><a class="reference internal" href="#id103" id="id285">Bug Fixes</a></li>
<li><a class="reference internal" href="#id104" id="id286">Known Issues</a></li>
</ul>
</li>
<li><a class="reference internal" href="#release-3-2-0" id="id287">Release 3.2.0</a><ul>
<li><a class="reference internal" href="#id105" id="id288">New Features</a></li>
<li><a class="reference internal" href="#id106" id="id289">Improvements</a></li>
<li><a class="reference internal" href="#id107" id="id290">Bug Fixes</a></li>
<li><a class="reference internal" href="#id108" id="id291">API Changes</a></li>
<li><a class="reference internal" href="#id109" id="id292">Deprecated and Removed Features</a></li>
<li><a class="reference internal" href="#id110" id="id293">Known Issues</a></li>
</ul>
</li>
<li><a class="reference internal" href="#release-3-1-0" id="id294">Release 3.1.0</a><ul>
<li><a class="reference internal" href="#id111" id="id295">New Features</a></li>
<li><a class="reference internal" href="#id112" id="id296">Improvements</a></li>
<li><a class="reference internal" href="#id113" id="id297">Bug Fixes</a></li>
<li><a class="reference internal" href="#id114" id="id298">Deprecated and Removed Features</a></li>
<li><a class="reference internal" href="#known-issues-310" id="id299">Known Issues</a></li>
</ul>
</li>
<li><a class="reference internal" href="#release-3-0-3" id="id300">Release 3.0.3</a><ul>
<li><a class="reference internal" href="#id116" id="id301">Bug Fix</a></li>
<li><a class="reference internal" href="#id117" id="id302">Known Issues</a></li>
</ul>
</li>
<li><a class="reference internal" href="#release-3-0-2" id="id303">Release 3.0.2</a><ul>
<li><a class="reference internal" href="#id118" id="id304">Bug Fixes</a></li>
<li><a class="reference internal" href="#id119" id="id305">Known Issues</a></li>
</ul>
</li>
<li><a class="reference internal" href="#release-3-0-1" id="id306">Release 3.0.1</a><ul>
<li><a class="reference internal" href="#id121" id="id307">New Features</a></li>
<li><a class="reference internal" href="#id122" id="id308">Improvements</a></li>
<li><a class="reference internal" href="#id123" id="id309">Bug Fixes</a></li>
<li><a class="reference internal" href="#known-issues-301" id="id310">Known Issues</a></li>
</ul>
</li>
<li><a class="reference internal" href="#release-3-0-0" id="id311">Release 3.0.0</a><ul>
<li><a class="reference internal" href="#id125" id="id312">New Features</a></li>
<li><a class="reference internal" href="#new-user-interface" id="id313">New User Interface</a></li>
<li><a class="reference internal" href="#id126" id="id314">Improvement</a></li>
<li><a class="reference internal" href="#id127" id="id315">Bug Fixes</a></li>
<li><a class="reference internal" href="#id128" id="id316">API Changes</a></li>
<li><a class="reference internal" href="#id129" id="id317">Deprecated and Removed Features</a></li>
<li><a class="reference internal" href="#known-issues-300" id="id318">Known Issues</a></li>
</ul>
</li>
<li><a class="reference internal" href="#release-2-8-0" id="id319">Release 2.8.0</a><ul>
<li><a class="reference internal" href="#general" id="id320">General</a></li>
<li><a class="reference internal" href="#id131" id="id321">New Features</a></li>
<li><a class="reference internal" href="#id132" id="id322">Improvement</a></li>
<li><a class="reference internal" href="#id133" id="id323">API Changes</a></li>
<li><a class="reference internal" href="#known-issues-280" id="id324">Known Issues</a></li>
</ul>
</li>
<li><a class="reference internal" href="#release-2-7-1" id="id325">Release 2.7.1</a><ul>
<li><a class="reference internal" href="#id135" id="id326">API Changes</a></li>
<li><a class="reference internal" href="#id136" id="id327">New Features</a></li>
<li><a class="reference internal" href="#known-issues-271" id="id328">Known Issues</a></li>
</ul>
</li>
<li><a class="reference internal" href="#release-2-6-1" id="id329">Release 2.6.1</a><ul>
<li><a class="reference internal" href="#cdap-bug-fixes" id="id330">CDAP Bug Fixes</a></li>
<li><a class="reference internal" href="#known-issues-261" id="id331">Known Issues</a></li>
</ul>
</li>
<li><a class="reference internal" href="#release-2-6-0" id="id332">Release 2.6.0</a><ul>
<li><a class="reference internal" href="#id139" id="id333">API Changes</a></li>
<li><a class="reference internal" href="#id140" id="id334">New Features</a></li>
<li><a class="reference internal" href="#id141" id="id335">CDAP Bug Fixes</a></li>
<li><a class="reference internal" href="#known-issues-260" id="id336">Known Issues</a></li>
</ul>
</li>
<li><a class="reference internal" href="#release-2-5-2" id="id337">Release 2.5.2</a><ul>
<li><a class="reference internal" href="#id143" id="id338">CDAP Bug Fixes</a></li>
<li><a class="reference internal" href="#other-changes" id="id339">Other Changes</a></li>
<li><a class="reference internal" href="#id144" id="id340">Known Issues</a></li>
</ul>
</li>
<li><a class="reference internal" href="#release-2-5-1" id="id341">Release 2.5.1</a><ul>
<li><a class="reference internal" href="#id145" id="id342">CDAP Bug Fixes</a></li>
<li><a class="reference internal" href="#id146" id="id343">Other Changes</a></li>
<li><a class="reference internal" href="#id147" id="id344">Known Issues</a></li>
</ul>
</li>
<li><a class="reference internal" href="#release-2-5-0" id="id345">Release 2.5.0</a><ul>
<li><a class="reference internal" href="#id148" id="id346">New Features</a></li>
<li><a class="reference internal" href="#major-cdap-bug-fixes" id="id347">Major CDAP Bug Fixes</a></li>
<li><a class="reference internal" href="#known-issues-250" id="id348">Known Issues</a></li>
</ul>
</li>
</ul>
</div>
<div class="section" id="release-6-1-1">
<h2><a class="reference external" href="http://docs.cask.co/cdap/6.1.1/index.html">Release 6.1.1</a><a class="headerlink" href="#release-6-1-1" title="Permalink to this headline">🔗</a></h2>
<div class="section" id="summary">
<h3>Summary<a class="headerlink" href="#summary" title="Permalink to this headline">🔗</a></h3>
<p>This release introduces a number of new features, improvements, and bug fixes to CDAP. Some of the main highlights of the release are:</p>
<ol class="arabic">
<li><dl class="first docutils">
<dt><strong>Pipeline improvements</strong></dt>
<dd><ul class="first last simple">
<li>Validation checks for plugins for early error detection and prevention</li>
<li>New widgets for better pipeline configurability</li>
<li>Wrangler ADLS connection</li>
</ul>
</dd>
</dl>
</li>
<li><dl class="first docutils">
<dt><strong>Field Level Lineage</strong></dt>
<dd><ul class="first last simple">
<li>New, intuitive UI for field level lineage</li>
<li>Field level lineage support for more plugins</li>
</ul>
</dd>
</dl>
</li>
<li><dl class="first docutils">
<dt><strong>Platform enhancements</strong></dt>
<dd><ul class="first last simple">
<li>Performance mprovements across the platform</li>
<li>Migration of more UI components from Angular to React</li>
</ul>
</dd>
</dl>
</li>
</ol>
</div>
<div class="section" id="new-features">
<h3>New Features<a class="headerlink" href="#new-features" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-16102">CDAP-16102</a> - Added field level lineage support for Error Transform</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-16037">CDAP-16037</a> - Added region support for google cloud plugins</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-15795">CDAP-15795</a> - New UI landing page</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-15789">CDAP-15789</a> - Allow plugin developers to define filters to show/hide properties based on custom plugin configuration logic.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-15787">CDAP-15787</a> - Introduced new FailureCollector apis for better user experience via contextual error messages</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-15767">CDAP-15767</a> - Added support for reading INT96 types in parquet file sources.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-15728">CDAP-15728</a> - New ConfigurationGroup component in UI</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-15723">CDAP-15723</a> - Added support for pipeline to run in shared vpc network</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-15619">CDAP-15619</a> - Stage level validation for plugin properties.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-15482">CDAP-15482</a> - Added a new REST endpoint that retrieves back all field lineage information about a dataset.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-15342">CDAP-15342</a> - Added support for bytes types in the bigquery sink</li>
</ul>
</div>
<div class="section" id="deprecation">
<h3>Deprecation<a class="headerlink" href="#deprecation" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-15917">CDAP-15917</a> - Removed the outdated Validator plugin</li>
</ul>
</div>
<div class="section" id="bug-fixes">
<h3>Bug Fixes<a class="headerlink" href="#bug-fixes" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-16193">CDAP-16193</a> - Fix the preview run state after JVM restarted</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-16146">CDAP-16146</a> - content type detection now uses case insensitive file extensions</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-16137">CDAP-16137</a> - Fixed bug that prevents users from navigating to pipeline studio (indicating system artifacts being loaded for a long time).</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-15973">CDAP-15973</a> - Fixed the dataproc provisioner to log the error message if the dataproc creation operation fails.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-15899">CDAP-15899</a> - Fixed a bug that caused pipeline startup to take longer than needed for cloud runs</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-15879">CDAP-15879</a> - Fixed regex usage in GCS and S3 source plugins.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-15878">CDAP-15878</a> - Fixed a bug with the Datastore source that was overly restrictive when validating the user provided schema</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-15809">CDAP-15809</a> - Fixing a bug which can cause a thread spinning in an infinite while loop due to multi thread consumers on a queue that allows a single consumer.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-15770">CDAP-15770</a> - Fixed a bug that caused pipeline failures when writing nullable byte fields as json.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-15757">CDAP-15757</a> - Fixed a bug that caused MapReduce and Spark logs to be missing for remote pipeline runs</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-15747">CDAP-15747</a> - Fixed a race condition that could cause a program to get stuck in the pending state when stopped in the pending state</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-15742">CDAP-15742</a> - Added some safeguards to prevent cloud pipeline runs from getting stuck in certain edge cases</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-15726">CDAP-15726</a> - Fixed a bug where secure macros were not evaluated in preview mode</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-15617">CDAP-15617</a> - Fixed a bug in the BigQuery source that cause automatic bucket creation to fail if the dataset is in a different project.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-15583">CDAP-15583</a> - Fix bug in new user tour on lower resolution screens</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-15554">CDAP-15554</a> - Fixed a bug that wrong resolution is used if a time range is specified for metrics query</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-15535">CDAP-15535</a> - Fixed an issue where BigQuery multi sink doesn’t work if using an Oracle database as a source.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-15498">CDAP-15498</a> - Fixed the dataproc provisioner to disable YARN pre-emptive container killing and to disable conscrypt.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-15445">CDAP-15445</a> - Fixed a bug in the MLPredictor plugin that caused error when using a classification model</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-15423">CDAP-15423</a> - Fixed bug that didn’t allow users to paste schema as runtime argument</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-15388">CDAP-15388</a> - Spark pipelines no longer try to run sinks in parallel unless runtime argument ‘pipeline.spark.parallel.sinks.enabled’ is set to ‘true’. This prevents pipeline sections from being re-processed in the majority of situations.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-15373">CDAP-15373</a> - Fixed the dataproc provisioner to handle networks that do not use automatic subnet creation</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-15353">CDAP-15353</a> - Fixed a Wrangler bug where the wrong jdbc driver would be used in some situations and where required classes could be unavailable.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-15221">CDAP-15221</a> - Fixed a bug about artifact version comparison</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-15206">CDAP-15206</a> - Fixed a bug that the rollup of the workflow lineage does not remove the local datasets.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-15097">CDAP-15097</a> - Expanding filename format that UI takes in when uploading artifacts.</li>
</ul>
</div>
<div class="section" id="improvements">
<h3>Improvements<a class="headerlink" href="#improvements" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-16110">CDAP-16110</a> - Fixed batch pipeline preview to read only the preview records instead of the full input.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-16069">CDAP-16069</a> - Greatly improved the time it takes to calculate field level lineage</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-15983">CDAP-15983</a> - Set Spark as the default execution engine for batch pipeline</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-15794">CDAP-15794</a> - Improved error message for csv, tsv, and delimited formats when the schema has fewer fields than the data</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-15782">CDAP-15782</a> - Added support to automatically fill field level lineage for plugins that do not emit any</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-15738">CDAP-15738</a> - Upgrades Nodejs version from 8.x to 10.16.2</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-15677">CDAP-15677</a> - Added support to restore preview status after restart</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-15659">CDAP-15659</a> - Route user directly to the pipeline’s detail page from pipeline card in Control Center.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-15489">CDAP-15489</a> - New user experience for log level selection.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-15265">CDAP-15265</a> - Added image version as a configuration setting to the dataproc provisioner</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-16076">CDAP-16076</a> - Improved the way pipelines with macros that are provided by intermediate stages run.</li>
</ul>
</div>
</div>
<div class="section" id="release-6-0-0">
<h2><a class="reference external" href="http://docs.cask.co/cdap/6.0.0/index.html">Release 6.0.0</a><a class="headerlink" href="#release-6-0-0" title="Permalink to this headline">🔗</a></h2>
<div class="section" id="id1">
<h3>Summary<a class="headerlink" href="#id1" title="Permalink to this headline">🔗</a></h3>
<p>This release introduces a number of new features, improvements, and bug fixes to CDAP. Some of the main highlights of the release are:</p>
<ol class="arabic">
<li><dl class="first docutils">
<dt><strong>Storage SPIs</strong></dt>
<dd><ul class="first last simple">
<li>Storage SPIs provide abstraction for all system storage used by CDAP so that CDAP is more portable across runtime enviroments - Hadoop or Hadoop-free environments.</li>
</ul>
</dd>
</dl>
</li>
<li><dl class="first docutils">
<dt><strong>Portable Runtime</strong></dt>
<dd><ul class="first last simple">
<li>Provide a runtime architecture for CDAP to support both Hadoop and Hadoopless environments, such as Kubernetes, in a distributed and secure fashion.</li>
</ul>
</dd>
</dl>
</li>
<li><dl class="first docutils">
<dt><strong>Pipeline Enhancements</strong></dt>
<dd><ul class="first last simple">
<li>Improve experience of building pipelines with the help of features such as copy &amp; paste and minimap of the pipeline.</li>
<li>Add support for more data types.</li>
</ul>
</dd>
</dl>
</li>
</ol>
</div>
<div class="section" id="id2">
<h3>New Features<a class="headerlink" href="#id2" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-14330">CDAP-14330</a> - Added Google Cloud Storage copy and move action plugins.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-14533">CDAP-14533</a> - New pipeline list user interface.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-14613">CDAP-14613</a> - Added minimap to pipeline canvas.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-14645">CDAP-14645</a> - Added support for running CDAP system services in Kubernetes environment.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-14657">CDAP-14657</a> - Added the ability to copy and paste a node in pipeline studio.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-15058">CDAP-15058</a> - Added the ability to limit the number of concurrent pipeline runs.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-15095">CDAP-15095</a> - Added support for toggling Stackdriver integration in Google Cloud Dataproc cluster.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-15256">CDAP-15256</a> - Added support for Numeric and Array types in Google BigQuery plugins.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-15339">CDAP-15339</a> - Added support for showing decimal field types in plugin schemas in pipeline view.</li>
</ul>
</div>
<div class="section" id="id3">
<h3>Improvements<a class="headerlink" href="#id3" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-13632">CDAP-13632</a> - Added support for CDH 5.15.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-14653">CDAP-14653</a> - Revamps top navbar for CDAP UI based on material design.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-14667">CDAP-14667</a> - Secure store supports integration with other KMS systems such as Google Cloud KMS using new Secure Store SPIs.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7208">CDAP-7208</a> - Improved CDAP Master logging of events related to programs that it launches.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-14343">CDAP-14343</a> - Use a shared thread pool for provisioning tasks to increase thread utilization.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-14569">CDAP-14569</a> - Improve performance of LevelDB backed Table implementation.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-14571">CDAP-14571</a> - Wrangler supports secure macros in connection.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-14617">CDAP-14617</a> - Significantly improve performance of Transactional Messaging System.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-14821">CDAP-14821</a> - Added early validation for the properties of the Google BigQuery sink to fail during pipeline deployment instead of at runtime.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-14823">CDAP-14823</a> - Improved the error message when a null value is read for a non-nullable field in avro file sources.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-15047">CDAP-15047</a> - Improved loading of system artifacts to load in parallel instead of sequentially.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-15059">CDAP-15059</a> - Improved Google Cloud Dataproc provisioner to allow configuring default projectID from CDAP configuration.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-15318">CDAP-15318</a> - Added support of using runtime arguments to pass in extra configurations for Google Cloud Dataproc provisioner.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-14579">CDAP-14579</a> - Added support for spaces in file path for Google Cloud Storage plugin.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-14897">CDAP-14897</a> - Google BigQuery source now validates schema when the pipeline is deployed.</li>
</ul>
</div>
<div class="section" id="id4">
<h3>Bug Fixes<a class="headerlink" href="#id4" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12211">CDAP-12211</a> - Fixed a casting bug for the DB source where unsigned integer column were incorrectly being treated as integers instead of longs.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-13410">CDAP-13410</a> - Removed the need for ZooKeeper for service discovery in remote runtime environment.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7230">CDAP-7230</a> - Fixed an issue with recording lineage for realtime sources.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12941">CDAP-12941</a> - Fixed dynamic Spark plugin to use appropriate context classloader for loading dynamic Spark code.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-13554">CDAP-13554</a> - Fixed a bug that caused MapReduce pipelines to fail when using too many macros.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-13982">CDAP-13982</a> - Fixed an issue that caused pipelines with too many macros to fail when running in MapReduce.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-14666">CDAP-14666</a> - Fixed an issue with publishing metadata changes for profile assignments.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-14691">CDAP-14691</a> - Fixed a bug that would cause workspace ids to clash when wrangling items of the same name.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-14702">CDAP-14702</a> - Fixed a bug in secure store caused by breaking changes in Java update 171. Users should be able to get secure keys on java 8u171.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-14708">CDAP-14708</a> - Fixed a bug that caused Google Cloud Dataproc clusters to fail provisioning if a firewall rule that denies ingress traffic existed in the project.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-14709">CDAP-14709</a> - Fixed a bug that would cause data preparation to fail when preparing a large file in Google Cloud Storage.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-14724">CDAP-14724</a> - Fixed a bug that caused action-only pipelines to fail when running using a cloud profile.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-14744">CDAP-14744</a> - Fixed an issue with adding business tags to an entity.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-14778">CDAP-14778</a> - Fixed an issue in handling metadata search parameters.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-14779">CDAP-14779</a> - Fixed a bug that would cause pipelines to fail on remote clusters if the very first pipeline run was an action-only pipeline.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-14857">CDAP-14857</a> - Fixed the standard deviation aggregate functions to work, even if there is only one element in a group.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-14951">CDAP-14951</a> - Fixed a bug in the Google BigQuery sink that would cause pipelines to fail when writing to a dataset in a different region.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-15001">CDAP-15001</a> - Fixed a race condition in processing profile assignments.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-15013">CDAP-15013</a> - Fixed an issue that could cause inconsistencies in metadata.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-15069">CDAP-15069</a> - Fixed an issue with displaying workspace metadata in the UI.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-15127">CDAP-15127</a> - Fixed a race condition in the remote runtime scp implementation that could cause process to hang.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-15196">CDAP-15196</a> - Fixed an issue with metadata search result pagination.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-15223">CDAP-15223</a> - Fixed Wrangler DB connection where a bad JDBC driver could stay in cache for 60 minutes, making DB connection not usable.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-15249">CDAP-15249</a> - Fixed a NullPointerException in Google Cloud Dataproc provision for when there was no network configured.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-15299">CDAP-15299</a> - Fixed a bug that caused some aggregator and joiner keys to be dropped if they hashed to the same value as another key.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-15332">CDAP-15332</a> - Fixed a bug in the RuntimeMonitor that doesn’t reconnect through SSH correctly, causing failure in monitoring the correct program state.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-15369">CDAP-15369</a> - Fixed Google Cloud Dataproc runtime for Google Cloud Platform projects where OS Login is enabled.</li>
</ul>
</div>
<div class="section" id="deprecated-and-removed-features">
<h3>Deprecated and Removed Features<a class="headerlink" href="#deprecated-and-removed-features" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-15241">CDAP-15241</a> - Deprecated HDFSMove and HDFSDelete plugins from core plugins.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-14591">CDAP-14591</a> - Removed Streams and Stream Views, which were deprecated in CDAP 5.0.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-14592">CDAP-14592</a> - Removed Flow, which was deprecated in CDAP 5.0.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-14529">CDAP-14529</a> - Removed deprecated HDFSSink Plugin.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-14772">CDAP-14772</a> - Removed the plugin endpoints feature to prevent execution of plugin code in the cdap master. Endpoints were only used for schema propagation, which has moved to the pipeline system service.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-14886">CDAP-14886</a> - Removed the support for custom routing for user services.</li>
</ul>
</div>
</div>
<div class="section" id="release-5-1-2">
<h2><a class="reference external" href="http://docs.cask.co/cdap/5.1.2/index.html">Release 5.1.2</a><a class="headerlink" href="#release-5-1-2" title="Permalink to this headline">🔗</a></h2>
<div class="section" id="id5">
<h3>Improvements<a class="headerlink" href="#id5" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-13430">CDAP-13430</a> - Improved performance of Apache Spark pipelines that write to multiple sinks.</li>
</ul>
</div>
<div class="section" id="id6">
<h3>Bug Fixes<a class="headerlink" href="#id6" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-14558">CDAP-14558</a> - Fixed a bug where pipeline checkpointing is always on regardless of the value set by the user in realtime pipeline.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-14578">CDAP-14578</a> - Fixed a bug where artifacts could not be uploaded through UI.</li>
</ul>
</div>
</div>
<div class="section" id="release-5-1-1">
<h2><a class="reference external" href="http://docs.cask.co/cdap/5.1.1/index.html">Release 5.1.1</a><a class="headerlink" href="#release-5-1-1" title="Permalink to this headline">🔗</a></h2>
<div class="section" id="id7">
<h3>Improvements<a class="headerlink" href="#id7" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-14490">CDAP-14490</a> - Google Cloud Spanner sink will create database and table if they do not exist.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-14542">CDAP-14542</a> - Added a Dataset Project config property to the Google BigQuery source to allow reading from a dataset in another project.</li>
</ul>
</div>
<div class="section" id="id8">
<h3>Bug Fixes<a class="headerlink" href="#id8" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12229">CDAP-12229</a> - Fixed an issue that caused avro, parquet, and orc classes across file, Google Cloud Storage, and S3 plugins to clash and cause pipeline failures.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-14511">CDAP-14511</a> - Fixed a bug where plugins that register other plugins would not use the correct id when using the PluginSelector API.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-14515">CDAP-14515</a> - Fixed a bug where upgraded CDAP instances were not able to load artifacts.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-14524">CDAP-14524</a> - Fixed an issue where configuration of sink was overwritten by source.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-14538">CDAP-14538</a> - Fixed a packaging bug in kafka-plugins that prevented the plugins from being visible.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-14549">CDAP-14549</a> - Fixed a bug where plugins created by other plugins would not have their macros evaluated.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-14560">CDAP-14560</a> - Removed LZO as a compression option for snapshot and time partitioned fileset sinks since the codec cannot be packaged with the plugin.</li>
</ul>
</div>
</div>
<div class="section" id="release-5-1-0">
<h2><a class="reference external" href="http://docs.cask.co/cdap/5.1.0/index.html">Release 5.1.0</a><a class="headerlink" href="#release-5-1-0" title="Permalink to this headline">🔗</a></h2>
<div class="section" id="id9">
<h3>Summary<a class="headerlink" href="#id9" title="Permalink to this headline">🔗</a></h3>
<p>This release introduces a number of new features, improvements and bug fixes to CDAP. Some of the main highlights of the release are:</p>
<ol class="arabic">
<li><dl class="first docutils">
<dt><strong>Date and Time Support</strong></dt>
<dd><ul class="first last simple">
<li>Support for Date, Time and Timestamp data types in the CDAP schema. In addition, this support is also now available in pipeline plugins and 数据预处理 directives.</li>
</ul>
</dd>
</dl>
</li>
<li><dl class="first docutils">
<dt><strong>Plugin Requirements</strong></dt>
<dd><ul class="first last simple">
<li>A way for plugins to specify certain runtime requirements, and the ability to filter available plugins based on those requirements.</li>
</ul>
</dd>
</dl>
</li>
<li><dl class="first docutils">
<dt><strong>Bootstrapping</strong></dt>
<dd><ul class="first last simple">
<li>A method to automatically bootstrap CDAP with a given state, such as a set of deployed apps, artifacts, namespaces, and preferences.</li>
</ul>
</dd>
</dl>
</li>
<li><dl class="first docutils">
<dt><strong>UI Customization</strong></dt>
<dd><ul class="first last simple">
<li>A way to customize the display of the CDAP UI by enabling or disabling certain features.</li>
</ul>
</dd>
</dl>
</li>
</ol>
</div>
<div class="section" id="id10">
<h3>New Features<a class="headerlink" href="#id10" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-14244">CDAP-14244</a> - Added support for Date/Time in Preparation. Also, added a new directive parse-timestamp to convert unix timestamp in long or string to Timestamp object.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-14245">CDAP-14245</a> - Added Date, Time, and Timestamp support in plugins (Wrangler, Google BigQuery, Google Cloud Spanner, Database).</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-14021">CDAP-14021</a> - Added Date, Time, and Timestamp support in CDAP Schema.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-14028">CDAP-14028</a> - Added Date, Time, and Timestamp support in UI.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-14053">CDAP-14053</a> - Added Google Cloud Spanner source and sink plugins in Pipeline and Google Cloud Spanner connection in Preparation.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-14185">CDAP-14185</a> - Added Google Cloud Pub/Sub realtime source.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-14088">CDAP-14088</a> - Added a new user onboarding tour to CDAP.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-13990">CDAP-13990</a> - Added the ability to customize UI through theme.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-14022">CDAP-14022</a> - Added a framework that can be used to bootstrap a CDAP instance.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-13746">CDAP-13746</a> - Added the ability to configure system wide provisioner properties that can be set by admins but not by users.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-13924">CDAP-13924</a> - Added capability to allow specifying requirements by plugins and filter them on the basis of their requirements.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-13975">CDAP-13975</a> - Added REST endpoints to query the run counts of a program.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-14260">CDAP-14260</a> - Added a REST endpoint to get the latest run record of multiple programs in a single call.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-13653">CDAP-13653</a> - Added support for Apache Spark 2.3.</li>
</ul>
</div>
<div class="section" id="id11">
<h3>Improvements<a class="headerlink" href="#id11" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-13566">CDAP-13566</a> - Improved runtime monitoring (which fetches program states, metadata and logs) of remotely launched programs from the CDAP Master by using dynamic port forwarding instead of HTTPS for communication.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-13977">CDAP-13977</a> - Removed duplicate classes to reduce the size of the sandbox by a couple hundred megabytes.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-14461">CDAP-14461</a> - Added cdap-env.sh to allow configuring jvm options while launching the Sandbox.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-14003">CDAP-14003</a> - Added support for bidirectional Field Level Lineage.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-14013">CDAP-14013</a> - Added capability for external dataset to record their schema.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-14091">CDAP-14091</a> - The Google Cloud Dataproc provisioner will try to pick up the project id and credentials from the environment if they are not specified.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-14104">CDAP-14104</a> - The Google Cloud Dataproc provisioner will use internal IP addresses when CDAP is in the same network as the Google Cloud Dataproc cluster.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-14168">CDAP-14168</a> - Added capability to always display current dataset schema in Field Level Lineage.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-13886">CDAP-13886</a> - Improved error handling in Preparation.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-14023">CDAP-14023</a> - Added a FileSink batch sink, FileMove action, and FileDelete action to replace their HDFS counterparts.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-14097">CDAP-14097</a> - Added a configurable jvm option to kill CDAP process immediately on sandbox when an OutOfMemory error occurs.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-14135">CDAP-14135</a> - Added better trace logging for dataset service.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-14386">CDAP-14386</a> - Make Google Cloud Storage, Google BigQuery, and Google Cloud Spanner connection properties optional (project id, service account keyfile path, temporary Google Cloud Storage bucket).</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-14401">CDAP-14401</a> - Google Cloud Pub/Sub sink will try to create the topic if it does not exist while preparing for the run.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-14475">CDAP-14475</a> - Added csv, tsv, delimited, json, and blob as formats to the S3 source and sink.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-14321">CDAP-14321</a> - Added csv, tsv, delimited, json, and blob as formats to the File source.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-9048">CDAP-9048</a> - Added a button on external sources and sinks to jump to the dataset detail page.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-14040">CDAP-14040</a> - Added format and suppress query params to the program logs endpoint to match the program run logs endpoint.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-14132">CDAP-14132</a> - Made all CDAP examples to be compatible with Spark 2.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-14220">CDAP-14220</a> - Added worker and master disk size properties to the Google Cloud Dataproc provisioner.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-14298">CDAP-14298</a> - Improved operational behavior of the dataset service.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-14372">CDAP-14372</a> - Fixed wrangler transform to make directives optional. If none are given, the transform is a no-op.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-14397">CDAP-14397</a> - Fixed Preparation to treat files wihtout extension as text files.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-14398">CDAP-14398</a> - Limited the number of files showed in S3 and Google Cloud Storage browser to 1000.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-14482">CDAP-14482</a> - Enhanced Google BigQuery sink to create dataset if the specified dataset does not exist.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-14489">CDAP-14489</a> - Increased log levels for the CDAP Sandbox so that only CDAP classes are at debug level.</li>
</ul>
</div>
<div class="section" id="id12">
<h3>Bug Fixes<a class="headerlink" href="#id12" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-14468">CDAP-14468</a> - Fixed the ‘distinct’ plugin to use a drop down for the list of fields and to have a button to get the output schema.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7444">CDAP-7444</a> - Ensured that destroy() is always called for MapReduce, even if initialize() fails.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-13008">CDAP-13008</a> - Fixed a bug where Alert Publisher will not work if there is a space in the label.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-13230">CDAP-13230</a> - Fixed a bug that caused Preparation to fail while parsing avro files.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-13878">CDAP-13878</a> - Fixed a misleading error message about hbase classes in cloud runtimes.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-13887">CDAP-13887</a> - Fixed a bug where the metric for failed profile program runs was not getting incremented when the run failed due to provisioning errors.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-13894">CDAP-13894</a> - Fixed a bug where querying metrics by time series will be incorrect after a certain amount of time.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-13959">CDAP-13959</a> - Fixed a bug where profile metrics is incorrect if an app is deleted.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-13965">CDAP-13965</a> - Fixed a deprovisioning bug when cluster creation would fail.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-13988">CDAP-13988</a> - Fixed an error where TMS publishing was retried indefinitely if the first attempt failed.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-14076">CDAP-14076</a> - Fixed a race condition in MapReduce that can cause a deadlock.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-14098">CDAP-14098</a> - Fixed a resource leak in preview feature.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-14107">CDAP-14107</a> - Fixed a bug that would cause RDD versions of the dynamic scala spark plugins to fail.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-14154">CDAP-14154</a> - Fixed a bug where profiles were getting applied to all program types instead of only workflows.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-14203">CDAP-14203</a> - Fixed a race condition by ensuring that a program is started before starting runtime monitoring for it.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-14211">CDAP-14211</a> - Fixed runs count for pipelines in UI to show correct number instead of limiting to 100.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-14223">CDAP-14223</a> - Fixed an issue where Google Cloud Dataproc client was not being closed, resulting in verbose error logs.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-14261">CDAP-14261</a> - Fixed a bug that could cause the provisioning state of stopped program runs to be corrupted.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-14271">CDAP-14271</a> - Fixed a bug that caused Preparation to be unable to list buckets in a Google Cloud Storage connection in certain environments.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-14303">CDAP-14303</a> - Fixed a bug where Google Cloud Dataproc provisioner is not able to provision a singlenode cluster.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-14390">CDAP-14390</a> - Fixed a bug where Preparation could not read json or xml files on Google Cloud Storage.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-14395">CDAP-14395</a> - Fixed Google Cloud Dataproc provisioner to use full API access scopes so that Google Cloud Spanner and Google Cloud Pub/Sub are accessible by default.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-14435">CDAP-14435</a> - Fixed a bug where profile metrics is not deleted when a profile is deleted.</li>
</ul>
</div>
<div class="section" id="id13">
<h3>Deprecated and Removed Features<a class="headerlink" href="#id13" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-14108">CDAP-14108</a> - Removed old and buggy dynamic spark plugins.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-14456">CDAP-14456</a> - Dropped support for MapR 4.1.</li>
</ul>
</div>
</div>
<div class="section" id="release-5-0-0">
<h2><a class="reference external" href="http://docs.cask.co/cdap/5.0.0/index.html">Release 5.0.0</a><a class="headerlink" href="#release-5-0-0" title="Permalink to this headline">🔗</a></h2>
<div class="section" id="id14">
<h3>Summary<a class="headerlink" href="#id14" title="Permalink to this headline">🔗</a></h3>
<ol class="arabic">
<li><dl class="first docutils">
<dt><strong>Cloud Runtime</strong></dt>
<dd><ul class="first last simple">
<li>Cloud Runtimes allow you to configure batch pipelines to run in a cloud environment.</li>
<li>Before the pipeline runs, a cluster is provisioned in the cloud. The pipeline is executed on that cluster, and the cluster is deleted after the run finishes.</li>
<li>Cloud Runtimes allow you to only use compute resources when you need them, enabling you to make better use of your resources.</li>
</ul>
</dd>
</dl>
</li>
<li><dl class="first docutils">
<dt><strong>Metadata</strong></dt>
<dd><ul class="first last">
<li><dl class="first docutils">
<dt><em>Metadata Driven Processing</em></dt>
<dd><ul class="first last simple">
<li>Annotate metadata to custom entities such as fields in a dataset, partitions of a dataset, files in a fileset</li>
<li>Access metadata from a program or plugin at runtime to facilitate metadata driven processing</li>
</ul>
</dd>
</dl>
</li>
<li><dl class="first docutils">
<dt><em>Field Level Lineage</em></dt>
<dd><ul class="first last simple">
<li>APIs to register operations being performed on fields from a program or a pipeline plugin</li>
<li>Platform feature to compute field level lineage based on operations</li>
</ul>
</dd>
</dl>
</li>
</ul>
</dd>
</dl>
</li>
<li><dl class="first docutils">
<dt><strong>Analytics</strong></dt>
<dd><ul class="first last simple">
<li>A simple, interactive, UI-driven approach to machine learning.</li>
<li>Lowers the bar for machine learning, allowing users of any level to understand their data and train models
while preserving the switches and levers that advanced users might want to tweak.</li>
</ul>
</dd>
</dl>
</li>
<li><dl class="first docutils">
<dt><strong>Operational Dashboard</strong></dt>
<dd><ul class="first last simple">
<li>A real-time interactive interface that visualizes program run statistics</li>
<li>Reporting for comprehensive insights into program runs over large periods of time</li>
</ul>
</dd>
</dl>
</li>
</ol>
</div>
<div class="section" id="id15">
<h3>New Features<a class="headerlink" href="#id15" title="Permalink to this headline">🔗</a></h3>
<div class="section" id="cloud-runtime">
<h4>Cloud Runtime<a class="headerlink" href="#cloud-runtime" title="Permalink to this headline">🔗</a></h4>
<ul class="simple">
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-13089">CDAP-13089</a> - Added Cloud Runtimes, which allow users to assign profiles to batch pipelines that control what environment the pipeline will run in. For each program run, a cluster in a cloud environment can be created for just that run, allowing efficient use of resources.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-13213">CDAP-13213</a> - Added a way for users to create compute profiles from UI to run programs in remote (cloud) environments using one of the available provisioners.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-13206">CDAP-13206</a> - Allowed users to specify a compute profile in UI to run the pipelines in cloud environments. Compute profiles can be specified either while running a pipeline manually or via a time schedule or via a pipeline state based trigger.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-13094">CDAP-13094</a> - Added a provisioner that allows users to run pipelines on Google Cloud Dataproc clusters.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-13774">CDAP-13774</a> - Added a provisioner that can run pipelines on remote Apache Hadoop clusters</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-13709">CDAP-13709</a> - Added an Amazon Elastic MapReduce provisioner that can run pipelines on AWS EMR.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-13380">CDAP-13380</a> - Added support for viewing logs in CDAP for programs executing using the Cloud Runtime.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-13432">CDAP-13432</a> - Added metadata such has pipelines, schedules and triggers that are associated with profiles. Also added metrics such as the total number of runs of a pipeline using a profile.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-13494">CDAP-13494</a> - Added the ability to disable and enable a profile</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-13276">CDAP-13276</a> - Added the capability to export or import compute profiles</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-13359">CDAP-13359</a> - Added the ability to set the default profile at namespace and instance levels.</li>
</ul>
</div>
<div class="section" id="metadata">
<h4>Metadata<a class="headerlink" href="#metadata" title="Permalink to this headline">🔗</a></h4>
<ul class="simple">
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-13260">CDAP-13260</a> - Added support for annotating metadata to custom entities. For example now a field in a dataset can be annotated with metadata.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-13264">CDAP-13264</a> - Added programmatic APIs for users to register field level operations from programs and plugins.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-13269">CDAP-13269</a> - Added REST APIs to retrieve the fields which were updated for a given dataset in a given time range, a summary of how those fields were computed, and details about operations which were responsible for updated those fields.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-13511">CDAP-13511</a> - Added the ability to view Field Level Lineage for datasets.</li>
</ul>
</div>
<div class="section" id="analytics">
<h4>Analytics<a class="headerlink" href="#analytics" title="Permalink to this headline">🔗</a></h4>
<ul class="simple">
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-13921">CDAP-13921</a> - Added CDAP Analytics as an interactive, UI-driver application that allows users to train machine learning models and use them in their pipelines to make predictions.</li>
</ul>
</div>
<div class="section" id="operational-dashboard">
<h4>Operational Dashboard<a class="headerlink" href="#operational-dashboard" title="Permalink to this headline">🔗</a></h4>
<ul class="simple">
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12865">CDAP-12865</a> - Added a Dashboard for real-time monitoring of programs and pipelines</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12901">CDAP-12901</a> - Added a UI to generate reports on programs and pipelines that ran over a period of time</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-13147">CDAP-13147</a> - Added feature to support Reports and Dashboard. Dashboard provides realtime status of program runs and future schedules. Reports is a tool for administrators to take a historical look at their applications program runs, statistics and performance</li>
</ul>
</div>
<div class="section" id="other-new-features">
<h4>Other New Features<a class="headerlink" href="#other-new-features" title="Permalink to this headline">🔗</a></h4>
<div class="section" id="id16">
<h5>数据流管道<a class="headerlink" href="#id16" title="Permalink to this headline">🔗</a></h5>
<ul class="simple">
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12839">CDAP-12839</a> - Added ‘Error’ and ‘Alert’ ports for plugins that support this functionality. To enable this functionality in your plugin, in addition to emitting alerts and errors from the plugin code, users have to set “emit-errors: true” and “emit-alerts: true” in their plugin json. Users can create connections from ‘Error’ port to Error Handlers plugins, and from ‘Alert’ port to Alert plugins</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-13045">CDAP-13045</a> - Added support for Apache Phoenix as a source in 数据流管道.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-13499">CDAP-13499</a> - Added support for Apache Phoenix database as a sink in 数据流管道.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12944">CDAP-12944</a> - Added the ability to support macro behavior for all widget types</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-13057">CDAP-13057</a> - Added the ability to view all the concurrent runs of a pipeline</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-13006">CDAP-13006</a> - Added the ability to view the runtime arguments, logs and other details of a particular run of a pipeline.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-13242">CDAP-13242</a> - Added UI support for Splitter plugins</li>
</ul>
</div>
<div class="section" id="id17">
<h5>数据预处理<a class="headerlink" href="#id17" title="Permalink to this headline">🔗</a></h5>
<ul class="simple">
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-13100">CDAP-13100</a> - Added a Google BigQuery connection for 数据预处理</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12880">CDAP-12880</a> - Added a point-and-click interaction to change the data type of a column in the 数据预处理 UI</li>
</ul>
</div>
<div class="section" id="miscellaneous">
<h5>Miscellaneous<a class="headerlink" href="#miscellaneous" title="Permalink to this headline">🔗</a></h5>
<ul class="simple">
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-13180">CDAP-13180</a> - Added a page to view and manage a namespace. Users can click on the current namespace card in the namespace dropdown to go the namespace’s detail page. In this page, they can see entities and profiles created in this namespace, as well as preferences, mapping and security configurations for this namespace.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12951">CDAP-12951</a> - Added the ability to restart CDAP programs to make it resilient to YARN outages.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-13242">CDAP-13242</a> - Implemented a new Administration page, with two tabs, Configuration and Management. In the Configuration tab, users can view and manage all namespaces, system preferences and system profiles. In the Management tab, users can get an overview of system services in CDAP and scale them.</li>
</ul>
</div>
</div>
</div>
<div class="section" id="id18">
<h3>Improvements<a class="headerlink" href="#id18" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-13280">CDAP-13280</a> - Added Spark 2 support for Kafka realtime source</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12727">CDAP-12727</a>, <a class="reference external" href="https://issues.cask.co/browse/CDAP-13068">CDAP-13068</a> - Added support for CDH 5.13 and 5.14.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-11805">CDAP-11805</a> - Added support for EMR 5.4 through 5.7</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-6308">CDAP-6308</a> - Upgraded CDAP Router to use Netty 4.1</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-13179">CDAP-13179</a> - Added support for automatically restarting long running program types (Service and Flow) upon application master process failure in YARN</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12549">CDAP-12549</a> - Added support for specifying custom consumer configs in Kafka source</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-13143">CDAP-13143</a> - Added support for specifying recursive schemas</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12275">CDAP-12275</a> - Added support to pass in YARN application ID in the logging context. This can help in correlating the ID of the program run in CDAP to the ID of the corresponding YARN application, thereby facilitating better debugging.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-9080">CDAP-9080</a> - Added the ability to deploy plugin artifacts without requiring a parent artifact. Such plugins are available for use in any parent artifacts</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12274">CDAP-12274</a> - Added the ability to import pipelines from the add entity modal (plus button)</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-11844">CDAP-11844</a> - Added the ability to save the runtime arguments of a pipeline as preferences, so that they do not have to be entered again.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12724">CDAP-12724</a> - Added the ability to specify dependencies to ScalaSparkCompute Action</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12426">CDAP-12426</a> - Added the ability to update the keytab URI for namespace’s impersonation configuration.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12279">CDAP-12279</a> - Added the ability to upload a User Defined Directive (UDD) using the plus button</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12963">CDAP-12963</a> - Allowed CDAP user programs to talk to Kerberos enabled HiveServer2 in the cluster without using a keytab</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-11096">CDAP-11096</a> - Allowed users to configure the transaction isolation level in database plugins</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-13573">CDAP-13573</a> - Configured sandbox to have secure store APIs enabled by default</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-13411">CDAP-13411</a> - Improved robustness of unit test framework by fixing flaky tests</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-13405">CDAP-13405</a> - Increased default twill reserved memory from 300mb to 768mb in order to prevent YARN from killing containers in standard cluster setups.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-13116">CDAP-13116</a> - Macro enabled all fields in the HTTP Callback plugin</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12974">CDAP-12974</a> - Removed concurrent upgrades of HBase coprocessors since it could lead to regions getting stuck in transit.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-13409">CDAP-13409</a> - Updated the CDAP sandbox to use Spark 2.1.0 as the default Spark version.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-13157">CDAP-13157</a> - Improved the documentation for defining Apache Ranger policies for CDAP entities</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12992">CDAP-12992</a> - Improved resiliency of router to zookeeper outages.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-13756">CDAP-13756</a> - Improved the performance of metadata upgrade by adding a dataset cache.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7644">CDAP-7644</a> - Added CLI command to fetch service logs</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12989">CDAP-12989</a> - Added rate limiting to router logs in the event of zookeeper outages</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-13759">CDAP-13759</a> - Renamed system metadata tables to v2.system.metadata_index.d, v2.system.metadata_index.i. and business metadata tables to v2.business.metadata_index.d, v2.business.metadata_index.i</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-6032">CDAP-6032</a> - Reduced CDAP Master’s local storage usage by deleting temporary directories created for programs as soon as programs are launched on the cluster.</li>
</ul>
</div>
<div class="section" id="id19">
<h3>Bug Fixes<a class="headerlink" href="#id19" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-13033">CDAP-13033</a> - Fixed a bug in TMS that prevented from correctly consuming multiple events emitted in the same transaction.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12875">CDAP-12875</a> - Fixed a bug that caused errors in the File source if it read parquet files that were not generated through Hadoop.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12693">CDAP-12693</a> - Fixed a bug that caused PySpark to fail to run with Spark 2 in local sandbox.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-13296">CDAP-13296</a> - Fixed a bug that could cause the status of a running program to be falsely returned as stopped if the run happened to change state in the middle of calculating the program state. Also fixed a bug where the state for a suspended workflow was stopped instead of running.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7052">CDAP-7052</a> - Fixed a bug that prevented MapReduce AM logs from YARN to show the right URI.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12973">CDAP-12973</a> - Fixed a bug that prevented Spark jobs from running after CDAP upgrade due to caching of jars.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-13026">CDAP-13026</a> - Fixed a bug that prevented a parquet snapshot source and sink to be used in the same pipeline</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-13593">CDAP-13593</a> - Fixed a bug that under some race condition, running a pipeline preview may cause the CDAP process to shut down.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12752">CDAP-12752</a> - Fixed a bug where a Spark program would fail to run when spark authentication is turned on</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-13123">CDAP-13123</a> - Fixed a bug where an ad-hoc exploration query on streams would fail in an impersonated namespace.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-13463">CDAP-13463</a> - Fixed a bug where pipelines with conditions on different branches could not be deployed.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12743">CDAP-12743</a> - Fixed a bug where the Scala Spark compiler had missing classes from classloader, causing compilation failure</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-13372">CDAP-13372</a> - Fixed a bug where the upgrade tool did not upgrade the owner meta table</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12647">CDAP-12647</a> - Fixed a bug with artifacts count, as when we we get artifact count from a namespace we also include system artifacts count causing the total artifact count to be much larger than real count.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-13364">CDAP-13364</a> - Fixed a class loading issue and a schema mismatch issue in the whole-file-ingest plugin.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12970">CDAP-12970</a> - Fixed a dependency bug that could cause HBase region servers to deadlock during a cold start</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12742">CDAP-12742</a> - Fixed an issue that caused pipeline failures if a Spark plugin tried to read or write a DataFrame using csv format.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-13532">CDAP-13532</a> - Fixed an issue that prevented user runtime arguments from being used in CDAP programs</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-13281">CDAP-13281</a> - Fixed an issue where Spark 2.2 batch pipelines with HDFS sinks would fail with delegation token issue error</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12731">CDAP-12731</a> - Fixed an issue with that caused hbase sink to fail when used alongside other sinks, using spark execution engine.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-13002">CDAP-13002</a> - Fixed an issue with the retrieval of non-ASCII strings from Table datasets.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-13040">CDAP-13040</a> - Fixed avro fileset plugins so that reserved hive keywords can be used as column names</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-13331">CDAP-13331</a> - Fixed macro enabled properties in plugin configuration to only have macro behavior if the entire value is a macro.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12988">CDAP-12988</a> - Fixed the logs REST API to return a valid json object when filters are specified</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-13110">CDAP-13110</a> - Fixes an issue where a dataset’s class loader was closed before the dataset itself, preventing the dataset from closing properly.</li>
</ul>
</div>
<div class="section" id="id20">
<h3>Deprecated and Removed Features<a class="headerlink" href="#id20" title="Permalink to this headline">🔗</a></h3>
<ul>
<li><p class="first"><a class="reference external" href="https://issues.cask.co/browse/CDAP-13721">CDAP-13721</a> - Deprecated the aggregation of metadata annotated with all the entities (application, programs, dataset, streams) associated in a run. From this release onwards metadata for program runs behaves like any other entity where a metadata can be directly annotated to it and retrieved from it. For backward compatibility, to achieve the new behavior an additional query parameter ‘runAggregation’ should be set to false while making the REST call to retrieve metadata of program runs.</p>
</li>
<li><p class="first"><a class="reference external" href="https://issues.cask.co/browse/CDAP-8141">CDAP-8141</a> - Dropped support for CDH 5.1, 5.2, 5.3 and HDP 2.0, 2.1 due to security vulnerabilities identified in them</p>
</li>
<li><p class="first"><a class="reference external" href="https://issues.cask.co/browse/CDAP-13493">CDAP-13493</a> - Removed HDFS, YARN, and HBase operational stats. These stats were not very useful, could generate confusing log warnings, and were confusing when used in conjunction with cloud profiles.</p>
</li>
<li><p class="first"><a class="reference external" href="https://issues.cask.co/browse/CDAP-13720">CDAP-13720</a> - Removed analytics plugins such as decision tree, naive bayes and logistic regression from Hub. The new Analytics flow in the UI should be used as a substitute for this functionality.</p>
</li>
<li><p class="first"><a class="reference external" href="https://issues.cask.co/browse/CDAP-12584">CDAP-12584</a> - Removed deprecated <code class="docutils literal notranslate"><span class="pre">cdap</span> <span class="pre">sdk</span></code> commands. Use <code class="docutils literal notranslate"><span class="pre">cdap</span> <span class="pre">sandbox</span></code> commands instead.</p>
</li>
<li><p class="first"><a class="reference external" href="https://issues.cask.co/browse/CDAP-13680">CDAP-13680</a> - Removed deprecated <code class="docutils literal notranslate"><span class="pre">cdap.sh</span></code> and <code class="docutils literal notranslate"><span class="pre">cdap-cli.sh</span></code> scripts.  Use <code class="docutils literal notranslate"><span class="pre">cdap</span> <span class="pre">sandbox</span></code> or <code class="docutils literal notranslate"><span class="pre">cdap</span> <span class="pre">cli</span></code> instead.</p>
</li>
<li><p class="first"><a class="reference external" href="https://issues.cask.co/browse/CDAP-11870">CDAP-11870</a> - Removed deprecated error datasets from pipelines. Error transforms should be used instead of error datasets, as they offer more functionality and flexibility.</p>
</li>
<li><p class="first"><a class="reference external" href="https://issues.cask.co/browse/CDAP-13353">CDAP-13353</a> - Deprecated HDFS Sink. Use the File sink instead.</p>
</li>
<li><p class="first"><a class="reference external" href="https://issues.cask.co/browse/CDAP-12692">CDAP-12692</a> - Removed deprecated stream size based schedules</p>
</li>
<li><p class="first"><a class="reference external" href="https://issues.cask.co/browse/CDAP-13419">CDAP-13419</a> - Deprecated streams and flows. Use Apache Kafka as a replacement technology for streams and spark streaming as a replacement technology for flows. Streams and flows will be removed in 6.0 release.</p>
</li>
<li><dl class="first docutils">
<dt><a class="reference external" href="https://issues.cask.co/browse/CDAP-5966">CDAP-5966</a> - Removed multiple deprecated programmatic and RESTful API’s in CDAP.</dt>
<dd><ul class="first last">
<li><dl class="first docutils">
<dt>Deprecated public APIs removed from the <code class="docutils literal notranslate"><span class="pre">cdap-api</span></code> module:</dt>
<dd><ul class="first last simple">
<li>Scheduling workflow using <code class="docutils literal notranslate"><span class="pre">io.cdap.cdap.api.schedule.Schedule</span></code> in <code class="docutils literal notranslate"><span class="pre">AbstractApplication</span></code> has been removed,
use <code class="docutils literal notranslate"><span class="pre">io.cdap.cdap.internal.schedule.ScheduleCreationSpec</span></code> for scheduling workflow.</li>
<li>Adding schedule using <code class="docutils literal notranslate"><span class="pre">io.cdap.cdap.api.schedule.Schedule</span></code> is removed in <code class="docutils literal notranslate"><span class="pre">ApplicationConfigurer</span></code>, use
<code class="docutils literal notranslate"><span class="pre">io.cdap.cdap.internal.schedule.ScheduleCreationSpec</span></code> for adding schedules.</li>
<li>Deprecated methods <code class="docutils literal notranslate"><span class="pre">getStreams</span></code>, <code class="docutils literal notranslate"><span class="pre">getDatasetModules</span></code>
and <code class="docutils literal notranslate"><span class="pre">getDatasetSpecs</span></code> have been removed from <code class="docutils literal notranslate"><span class="pre">FlowletDefinition</span></code>.</li>
<li><code class="docutils literal notranslate"><span class="pre">beforeSubmit</span></code> and <code class="docutils literal notranslate"><span class="pre">onFinish</span></code> methods have been removed from <code class="docutils literal notranslate"><span class="pre">Mapreduce</span></code> and <code class="docutils literal notranslate"><span class="pre">Spark</span></code> interfaces, use
<code class="docutils literal notranslate"><span class="pre">ProgramLifecycle#initialize</span></code> and <code class="docutils literal notranslate"><span class="pre">ProgramLifecycle#destroy</span></code> instead.</li>
<li><code class="docutils literal notranslate"><span class="pre">RunConstraints</span></code>, <code class="docutils literal notranslate"><span class="pre">ScheduleSpecification</span></code> and <code class="docutils literal notranslate"><span class="pre">Schedule</span></code> classes in package
<code class="docutils literal notranslate"><span class="pre">io.cdap.cdap.api.schedule</span></code> have been removed.</li>
<li><code class="docutils literal notranslate"><span class="pre">WorkflowAction</span></code>, <code class="docutils literal notranslate"><span class="pre">WorkflowActionConfigurer</span></code>, <code class="docutils literal notranslate"><span class="pre">WorkflowActionSpecification</span></code>, <code class="docutils literal notranslate"><span class="pre">AbstractWorkflowAction</span></code>
have been removed from the package <code class="docutils literal notranslate"><span class="pre">io.cdap.cdap.api.workflow</span></code>. Use <code class="docutils literal notranslate"><span class="pre">CustomAction</span></code> for workflows instead.</li>
<li><code class="docutils literal notranslate"><span class="pre">WorkflowConfigurer#addAction(WorkflowAction</span> <span class="pre">action)</span></code> has been removed, use
<code class="docutils literal notranslate"><span class="pre">addAction(CustomAction</span> <span class="pre">action)</span></code> instead.</li>
<li><code class="docutils literal notranslate"><span class="pre">MapReduceTaskContext#getInputName</span></code> has been removed, use <code class="docutils literal notranslate"><span class="pre">getInputContext</span></code> instead.</li>
</ul>
</dd>
</dl>
</li>
<li><dl class="first docutils">
<dt>The following deprecations have been removed from the <code class="docutils literal notranslate"><span class="pre">cdap-proto</span></code> module:</dt>
<dd><ul class="first last simple">
<li><code class="docutils literal notranslate"><span class="pre">ApplicationDetail#getArtifactVersion</span></code> has been removed, use <code class="docutils literal notranslate"><span class="pre">ApplicationDetail#getArtifact</span></code> instead.</li>
<li><code class="docutils literal notranslate"><span class="pre">getId()</span></code> method has been removed in <code class="docutils literal notranslate"><span class="pre">ApplicationRecord</span></code>, <code class="docutils literal notranslate"><span class="pre">DatasetRecord</span></code>, <code class="docutils literal notranslate"><span class="pre">ProgramLiveInfo</span></code> and
<code class="docutils literal notranslate"><span class="pre">ProgramRecord</span></code>.</li>
<li><code class="docutils literal notranslate"><span class="pre">Id</span></code> class has been removed.</li>
<li><code class="docutils literal notranslate"><span class="pre">ScheduleUpdateDetail</span></code> has been removed, use <code class="docutils literal notranslate"><span class="pre">ScheduleDetail</span></code> instead.</li>
<li><code class="docutils literal notranslate"><span class="pre">ScheduleType</span></code> has been removed, use <code class="docutils literal notranslate"><span class="pre">Trigger</span></code> instead.</li>
<li>Methods for getting <code class="docutils literal notranslate"><span class="pre">ScheduleSpecification</span></code> - <code class="docutils literal notranslate"><span class="pre">toScheduleSpec()</span></code> and
<code class="docutils literal notranslate"><span class="pre">toScheduleSpecs(List&lt;ScheduleDetail&gt;</span> <span class="pre">details)</span></code>, have been removed from <code class="docutils literal notranslate"><span class="pre">ScheduleDetail</span></code>.</li>
<li>Deprecated <code class="docutils literal notranslate"><span class="pre">MetadataRecord</span></code> class has been removed.</li>
</ul>
</dd>
</dl>
</li>
<li><dl class="first docutils">
<dt>The following deprecations have been removed from the <code class="docutils literal notranslate"><span class="pre">cdap-client</span></code> module:</dt>
<dd><ul class="first last simple">
<li>Removed methods which were using the old <code class="docutils literal notranslate"><span class="pre">io.cdap.cdap.proto.Id</span></code> classes in <code class="docutils literal notranslate"><span class="pre">ApplicationClient</span></code>,
<code class="docutils literal notranslate"><span class="pre">ArtifactClient</span></code>, <code class="docutils literal notranslate"><span class="pre">ClientConfig</span></code>, <code class="docutils literal notranslate"><span class="pre">DatsetClient</span></code>, <code class="docutils literal notranslate"><span class="pre">DatasetModuleClient</span></code>, <code class="docutils literal notranslate"><span class="pre">DatasetTypeClient</span></code>,
<code class="docutils literal notranslate"><span class="pre">LineageClient</span></code>, <code class="docutils literal notranslate"><span class="pre">MetricsClient</span></code>, <code class="docutils literal notranslate"><span class="pre">ProgramClient</span></code>, <code class="docutils literal notranslate"><span class="pre">ScheduleClient</span></code>, <code class="docutils literal notranslate"><span class="pre">ServiceClient</span></code>,
<code class="docutils literal notranslate"><span class="pre">StreamClient</span></code>, <code class="docutils literal notranslate"><span class="pre">StreamViewClient</span></code> and <code class="docutils literal notranslate"><span class="pre">WorkflowClient</span></code>.</li>
<li>Removed methods to add and update schedules using <code class="docutils literal notranslate"><span class="pre">ScheduleInstanceConfiguration</span></code> in <code class="docutils literal notranslate"><span class="pre">ScheduleClient</span></code>,
use methods accepting <code class="docutils literal notranslate"><span class="pre">ScheduleDetail</span></code> as parameter instead.</li>
</ul>
</dd>
</dl>
</li>
<li><p class="first">The REST API to get workflow status using <code class="docutils literal notranslate"><span class="pre">current</span></code> endpoint has been removed, use the workflow node state
endpoint <code class="docutils literal notranslate"><span class="pre">/nodes/state</span></code> instead to get workflow status.</p>
</li>
</ul>
</dd>
</dl>
</li>
</ul>
</div>
<div class="section" id="known-issues">
<h3>Known Issues<a class="headerlink" href="#known-issues" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-13853">CDAP-13853</a> - Updating the compute profile to use to manually run a pipeline using the UI can remove the
existing schedules and triggers of the pipeline.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-13919">CDAP-13919</a> - The reports feature does not work with Apache Spark 2.0 currently. As a workaround,
upgrade to use Spark version 2.1 or later to use reports.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-13896">CDAP-13896</a> - Plugins that are not supported while running a pipeline using a cloud runtime throw
unclear error messages at runtime.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-13274">CDAP-13274</a> - While some built-in plugins have been updated to emit operations for capturing field level
lineage, a number of them do not yet emit these operations.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-13326">CDAP-13326</a> - Pipelines cannot propagate dynamic schemas at runtime.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-13963">CDAP-13963</a> - Reading metadata is not supported when pipelines or programs run using a cloud runtime.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-13971">CDAP-13971</a> - Creating a pipeline from 数据预处理 when using an Apache Kafka plugin fails. As
a workaround, after clicking the Create Pipeline button, manually update the schema of the Kafka plugin to set a
single field named body as a non-nullable string.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-13910">CDAP-13910</a> - Metadata for custom entities is not deleted if it’s nearest known ancestor entity
(parent) is deleted.</li>
</ul>
</div>
</div>
<div class="section" id="release-4-3-4">
<h2><a class="reference external" href="http://docs.cask.co/cdap/4.3.4/index.html">Release 4.3.4</a><a class="headerlink" href="#release-4-3-4" title="Permalink to this headline">🔗</a></h2>
<div class="section" id="id21">
<h3>Improvements<a class="headerlink" href="#id21" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-13116">CDAP-13116</a> - Macro enabled all fields in the HTTP Callback plugin</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-13119">CDAP-13119</a> - Optimized the planner to reduce the amount of temporary data required in certain types of mapreduce pipelines.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-13122">CDAP-13122</a> - Minor optimization to reduce the number of mappers used to read intermediate data in mapreduce pipelines</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-13139">CDAP-13139</a> - Improves the schema generation for database sources.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-13179">CDAP-13179</a> - Automatic restart of long running program types (Service and Flow) upon application master process failure in YARN</li>
</ul>
</div>
<div class="section" id="id22">
<h3>Bug Fixes<a class="headerlink" href="#id22" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12875">CDAP-12875</a> - Fixed a bug that caused errors in the File source if it read parquet files that were not generated through Hadoop.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-13110">CDAP-13110</a> - Fixed an issue where a dataset’s class loader was closed before the dataset itself, preventing the dataset from closing properly.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-13120">CDAP-13120</a> - Fixed a bug that caused directories to be left around if a workflow used a partitioned fileset as a local dataset</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-13123">CDAP-13123</a> - Fixed a bug that caused a hive Explore query on Streams to not work.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-13129">CDAP-13129</a> - Fixed a planner bug to ensure that sinks are never placed in two different mapreduce phases in the same pipeline.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-13158">CDAP-13158</a> - Fixed a race condition when running multiple spark programs concurrently at a Workflow fork that can lead to workflow failure</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-13171">CDAP-13171</a> - Fixed an issue with creating a namespace if the namespace principal is not a member of the namespace home’s group.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-13191">CDAP-13191</a> - Fixed a bug that caused completed run records to be missed when storing run state, resulting in misleading log messages about ignoring killed states.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-13192">CDAP-13192</a> - Fixed a bug in FileBatchSource that prevented ignoreFolders property from working with avro and parquet inputs</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-13205">CDAP-13205</a> - Fixed an issue where inconsistencies in the schedulestore caused scheduler service to keep exiting.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-13217">CDAP-13217</a> - Fixed an issue that would cause changes in program state to be ignored if the program no longer existed, resulting in the run record corrector repeatedly failing to correct run records</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-13218">CDAP-13218</a> - Fixed the state of Workflow, MapReduce, and Spark program to be reflected correctly as KILLED state when user explicitly terminated the running program</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-13223">CDAP-13223</a> - Fixed directive syntaxes in point and click interactions for some date formats</li>
</ul>
</div>
</div>
<div class="section" id="release-4-3-3">
<h2><a class="reference external" href="http://docs.cask.co/cdap/4.3.3/index.html">Release 4.3.3</a><a class="headerlink" href="#release-4-3-3" title="Permalink to this headline">🔗</a></h2>
<div class="section" id="id23">
<h3>Improvements<a class="headerlink" href="#id23" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12942">CDAP-12942</a> - GroupBy aggregator plugin fields are now macro enabled.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12963">CDAP-12963</a> - Allow CDAP user programs to talk to Kerberos enabled HiveServer2 in the cluster without using a keytab.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12974">CDAP-12974</a> - Removed concurrent upgrades of HBase coprocessors since it could lead to regions getting stuck in transit.</li>
</ul>
</div>
<div class="section" id="id24">
<h3>Bug Fixes<a class="headerlink" href="#id24" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7052">CDAP-7052</a> - Fixed a bug that prevented MapReduce AM logs from YARN to show the right URI.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7644">CDAP-7644</a> - Added CLI command to fetch service logs.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12774">CDAP-12774</a> - Increased the dataset changeset size and limit to integer max by default.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12900">CDAP-12900</a> - Fixed a bug where macro for output schema of a node was not saved when the user closed the node properties modal.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12930">CDAP-12930</a> - Fixed a bug where explore queries would fail against paths in HDFS encryption zones, for certain Hadoop distributions.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12945">CDAP-12945</a> - Fixed a bug where the old connection is not removed from the pipeline config when you move the connection’s pointer to another node.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12946">CDAP-12946</a> - Fixed a bug in the pipeline planner where pipelines that used an action before multiple sources would either fail to deploy or deploy with an incorrect plan.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12970">CDAP-12970</a> - Fixed a dependency bug that could cause HBase region servers to deadlock during a cold start.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-13002">CDAP-13002</a> - Fixed an issue with the retrieval of non-ASCII strings from Table datasets.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-13021">CDAP-13021</a> - Messaging table coprocessor now gets upgraded when the underlying HBase version is changed without any change in the CDAP version.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-13026">CDAP-13026</a> - Fixed a bug that prevented a parquet snapshot source and sink to be used in the same pipeline.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-13033">CDAP-13033</a> - Fixed a bug in TMS that prevented correctly consuming multiple events emitted in the same transaction.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-13037">CDAP-13037</a> - Make TransactionContext resilient against getTransactionAwareName() failures.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-13040">CDAP-13040</a> - Fixed avro fileset plugins so that reserved hive keywords can be used as column names.</li>
</ul>
</div>
</div>
<div class="section" id="release-4-3-2">
<h2><a class="reference external" href="http://docs.cask.co/cdap/4.3.2/index.html">Release 4.3.2</a><a class="headerlink" href="#release-4-3-2" title="Permalink to this headline">🔗</a></h2>
<div class="section" id="id25">
<h3>New Features<a class="headerlink" href="#id25" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12771">CDAP-12771</a> - Added GCS connection to Data Prep.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12018">CDAP-12018</a> - Added S3 connection to Data Prep.</li>
</ul>
</div>
<div class="section" id="id26">
<h3>Improvements<a class="headerlink" href="#id26" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-11805">CDAP-11805</a> - Added support for EMR 5.4 through 5.7.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12727">CDAP-12727</a> - Added support for CDH 5.13.0.</li>
</ul>
</div>
<div class="section" id="id27">
<h3>Bug Fixes<a class="headerlink" href="#id27" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li><a class="reference external" href="https://issues.cask.co/browse/CCDAP-6032">CCDAP-6032</a> - Minimize master’s local storage usage by deleting the temporary directories created on the cdap-master for programs as soon as programs are launched on the cluster.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12682">CDAP-12682</a> - Fixed an issue where UI was looking for the wrong property for SSL port.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12693">CDAP-12693</a> - Fixed a bug that causes PySpark to fail to run with Spark 2 in local sandbox.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12701">CDAP-12701</a> - Fixed a bug that causes deployment of pipelines with condition plugins to fail on Apache Ambari clusters.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12731">CDAP-12731</a> - Fixed an issue that caused HBase Sink to fail when used alongside other sinks, using the Spark execution engine.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12743">CDAP-12743</a> - Fixed a bug where the Scala Spark compiler has missing classes from Classloader, causing compilation failure.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12752">CDAP-12752</a> - Fixed a bug where Spark programs failed to run when Spark authentication is turned on.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12769">CDAP-12769</a> - Fixed an issue with running the dynamic Scala Spark plugin on Windows. Directory which is used to store the compiled scala classes now contains ‘.’ as a separator instead of ‘:’ which was causing failure on Windows machines.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12843">CDAP-12843</a> - Fixed an issue that prevented auto-fill of schema for Datasets created by an ORC sink plugin.</li>
</ul>
</div>
</div>
<div class="section" id="release-4-3-1">
<h2><a class="reference external" href="http://docs.cask.co/cdap/4.3.1/index.html">Release 4.3.1</a><a class="headerlink" href="#release-4-3-1" title="Permalink to this headline">🔗</a></h2>
<div class="section" id="id28">
<h3>New Features<a class="headerlink" href="#id28" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12592">CDAP-12592</a> - Adds new visualization tool to give insights about data prepped up in data preparation tool.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12620">CDAP-12620</a> - Adds a way to trigger invalid transaction pruning via a REST endpoint.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12595">CDAP-12595</a> - Adds UI to make HTTP request in CDAP.</li>
</ul>
</div>
<div class="section" id="id29">
<h3>Improvements<a class="headerlink" href="#id29" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12598">CDAP-12598</a> - Added a downgrade command to the pipeline upgrade tool, allowing users to downgrade pipelines to a previous version.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12541">CDAP-12541</a> - Improved memory usage of data pipeline with joiner in mapreduce execution engine.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12176">CDAP-12176</a> - Added ability to select/clear all the checkboxes for Provided runtime arguments</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12646">CDAP-12646</a> - Fixed a performance issue with the run record corrector</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12380">CDAP-12380</a> - Added a capability to configure program containers memory settings through runtime arguments and preferences</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-8499">CDAP-8499</a> - Applies the extra jvm options configuration to all task containers in MapReduce</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12546">CDAP-12546</a> - Fixed a classloader leakage issue when PySpark is used in local sandbox</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12593">CDAP-12593</a> - Ability to list the datasets based on the set of dataset properties</li>
</ul>
</div>
<div class="section" id="id30">
<h3>Bug Fixes<a class="headerlink" href="#id30" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12645">CDAP-12645</a> - MapReduce Task-related metrics will be emitted from individual tasks instead of MapReduce driver.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12628">CDAP-12628</a> - Fixed the filter if missing flow in the UI to also apply on null values in addition to empty values.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12612">CDAP-12612</a> - Fixed the fill-null-or-empty directive to allow spaces in the default value</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12588">CDAP-12588</a> - Fixed a bug that authorization cannot be turned on if kerberos is disabled</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12578">CDAP-12578</a> - Fixed an issue that caused the pipeline upgrade tool to upgrade pipelines in a way that would cause UI failures when the upgraded pipeline is viewed.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12577">CDAP-12577</a> - Spark compat directories in the system artifact directory will now be automatically checked, regardless of whether they are explicitly set in app.artifacts.dir.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12570">CDAP-12570</a> - Added option to enable/disable emitting program metrics and option to include or skip task level information in metrics  context. This option can be used with scoping at program and program-type level similar to setting system resources with scoping.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12569">CDAP-12569</a> - Improved error messaging when there is an error while in publishing metrics in MetricsCollection service.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12567">CDAP-12567</a> - Fixed a bug that CDAP is not able to clean up and aggregate on streams in an authorization enabled environment</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12559">CDAP-12559</a> - Fixed log message format to include class name and line number when logged in master log</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12526">CDAP-12526</a> - Adds various improvements to the transaction system, including the ability to limit the size of a transaction’s change set; better insights into the cause of transaction conflicts; improved concurrency when writing to the transaction log;  better handling of border conditions during invalid transaction pruning; and ease of use for the transaction pruning diagnostic tool.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12495">CDAP-12495</a> - Fix the units for YARN memory stats on Administration UI page.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12482">CDAP-12482</a> - Fixed a bug where the app detail contains entity information that the user does not have any privilege on</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12476">CDAP-12476</a> - Fixed preview results for pipelines with condition stages</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12457">CDAP-12457</a> - Fixed a bug that caused failures for Hive queries using MR execution engine in CM 5.12 clusters.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12454">CDAP-12454</a> - Fixes an issue where transaction coprocessors could sometimes not access their configuration.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12451">CDAP-12451</a> - UI: Add ability to view payload configuration of pipeline triggers</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12441">CDAP-12441</a> - Fixed a bug that the cache timeout was not changed with the value of <code class="docutils literal notranslate"><span class="pre">security.authorization.cache.ttl.secs</span></code></li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12415">CDAP-12415</a> - Fixed an issue with not able to use HiveContext in Spark</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12387">CDAP-12387</a> - Added the authorization policy for adding/deleting schedules</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12377">CDAP-12377</a> - Fixes an issue where the transaction service could hang during shutdown.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12333">CDAP-12333</a> - Fixed issue where loaded data was not consistently rendering when navigating to 数据预处理 from other parts of CDAP.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12314">CDAP-12314</a> - Improves the performance of HBase operations when there are many invalid transactions.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12240">CDAP-12240</a> - Improves a previously misleading log message.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7651">CDAP-7651</a> - Fixed an issue that hive query may failed if the configuration has too many variables substitution.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7243">CDAP-7243</a> - Added mechanism to clean up local dataset if the workflows creating them are killed</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7049">CDAP-7049</a> - Improved the error message in the case that a kerberos principal is deleted or keytab is invalid, during impersonation.</li>
</ul>
</div>
</div>
<div class="section" id="release-4-3-0">
<h2><a class="reference external" href="http://docs.cask.co/cdap/4.3.0/index.html">Release 4.3.0</a><a class="headerlink" href="#release-4-3-0" title="Permalink to this headline">🔗</a></h2>
<div class="section" id="id31">
<h3>Summary<a class="headerlink" href="#id31" title="Permalink to this headline">🔗</a></h3>
<ol class="arabic">
<li><dl class="first docutils">
<dt><strong>数据流管道:</strong></dt>
<dd><ul class="first last simple">
<li>Support for conditional execution of parts of a pipeline</li>
<li>Ability for pipelines to trigger other pipelines for cross-team, cross-pipeline inter-connectivity, and to build complex interconnected pipelines.</li>
<li>Improved pipeline studio with redesigned nodes, undo/redo capability, metrics</li>
<li>Automated upgrade of pipelines to newer CDAP versions</li>
<li>Custom icons and labels for pipeline plugins</li>
<li>Operational insights into pipelines</li>
</ul>
</dd>
</dl>
</li>
<li><dl class="first docutils">
<dt><strong>数据预处理:</strong></dt>
<dd><ul class="first last simple">
<li>Support for User Defined Directives (UDD), so users can write their own custom directives for cleansing/preparing data.</li>
<li>Restricting Directive Usage and ability to alias Directives for your IT Administrators to control directive access</li>
</ul>
</dd>
</dl>
</li>
<li><dl class="first docutils">
<dt><strong>Governance &amp; Security:</strong></dt>
<dd><ul class="first last simple">
<li>Standardized authorization model</li>
<li>Apache Ranger Integration for authorization of CDAP entities</li>
</ul>
</dd>
</dl>
</li>
<li><dl class="first docutils">
<dt><strong>Enhanced support for Apache Spark:</strong></dt>
<dd><ul class="first last simple">
<li>PySpark Support so data scientists can develop their Spark logic in Python, while still taking advantage of enterprise integration capabilities of CDAP</li>
<li>Spark Dataframe Support so Spark developers can access CDAP datasets as Spark DataFrames</li>
</ul>
</dd>
</dl>
</li>
<li><dl class="first docutils">
<dt><strong>New Frameworks and Tools:</strong></dt>
<dd><ul class="first last simple">
<li>Microservices for real-time IoT use cases.</li>
<li>Distributed Rules Engine - for Business Analysts to effectively manage rules for data transformation and data policy</li>
</ul>
</dd>
</dl>
</li>
</ol>
</div>
<div class="section" id="id32">
<h3>New Features<a class="headerlink" href="#id32" title="Permalink to this headline">🔗</a></h3>
</div>
<div class="section" id="enhancements">
<h3>数据流管道 Enhancements<a class="headerlink" href="#enhancements" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12033">CDAP-12033</a> - Added a new splitter transform plugin type that can send output to different ports. Also added a union splitter transform that will send records to different ports depending on which type in the union it is and a splitter transform that splits records based on whether the specified field is null.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12034">CDAP-12034</a> - Added a way for pipeline plugins to emit alerts, and a new AlertPublisher plugin type that publishes those alerts. Added a plugin that publishes alerts to CDAP TMS and an Apache Kafka Alert Publisher plugin to publish alerts to a Kafka topic.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12108">CDAP-12108</a> - Batch data pipelines now support condition plugin types which can control the flow of execution of the pipeline. Condition plugins in the pipeline have access to the stage statistics such as number of input records, number of output records, number of error records generated from the stages which executed prior to the condition node. Also implemented Apache Commons JEXL based condition plugin which is available by default for the batch data pipelines.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12167">CDAP-12167</a> - Plugin <code class="docutils literal notranslate"><span class="pre">prepareRun</span></code> and <code class="docutils literal notranslate"><span class="pre">onFinish</span></code> methods now run in a separate transaction per plugin so that pipelines with many plugins will not timeout.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12191">CDAP-12191</a> - All pipeline plugins now have access to the pipeline namespace and name through their context object.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-9107">CDAP-9107</a> - Added a feature that allows undoing and redoing of actions in pipeline Studio.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12057">CDAP-12057</a> - Made pipeline nodes bigger to show the version and metrics on the node.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12077">CDAP-12077</a> - Revamped pipeline connections, to allow dropping a connection anywhere on the node, and allow selecting and deleting multiple connections using the Delete key.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-10619">CDAP-10619</a> - Added an automated UI flow for users to upgrade pipelines to newer CDAP versions.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-11889">CDAP-11889</a> - Added visualization for pipeline in UI. This helps visualizing runs, logs/warnings and data flowing through each node for each run in the pipeline.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12111">CDAP-12111</a> - Added support for plugins of plugins. This allows the parent plugin to expose some APIs that its own plugins will implement and extend.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12114">CDAP-12114</a> - Added ability to support custom label and custom icons for pipeline plugins.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-10974">CDAP-10974</a> - BatchSource, BatchSink, BatchAggregator, BatchJoiner, and Transform plugins now have a way to get SettableArguments when preparing a run, which allows them to set arguments for the rest of the pipeline.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-10653">CDAP-10653</a> - Runtime arguments are now available to the script plugins such as Javascript and Python via the Context object.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12472">CDAP-12472</a> - Added a method to PluginContext that will return macro evaluated plugin properties.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12094">CDAP-12094</a> - Enhanced add field transform plugin to add multiple fields</li>
</ul>
</div>
<div class="section" id="triggers">
<h3>Triggers<a class="headerlink" href="#triggers" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-11912">CDAP-11912</a> - Added capabilities to trigger programs and data pipelines based on status of other programs and data pipelines.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12382">CDAP-12382</a> - Added the capability to use plugin properties and runtime arguments from the triggering data pipeline as runtime arguments in the triggered data pipeline.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12232">CDAP-12232</a> - Added composite AND and OR trigger.</li>
</ul>
</div>
<div class="section" id="id33">
<h3>数据预处理 Enhancements<a class="headerlink" href="#id33" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-11618">CDAP-11618</a> - Added the ability for users to connect 数据预处理 to their existing data in Apache Kafka.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12092">CDAP-12092</a> - Added point and click interaction for performing various calculations on data in Data Prep.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12118">CDAP-12118</a> - Added point and click interaction for applying custom transformations in Data Prep.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-9530">CDAP-9530</a> - Added point and click interaction to mask column data.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-9532">CDAP-9532</a> - Added point and click interaction to encode/decode column data</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-11869">CDAP-11869</a> - Added point and click interaction to parse Avro and Excel files.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-11977">CDAP-11977</a> - Added point and click interaction for replacing column names in bulk.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12091">CDAP-12091</a> - Added point and click interaction for defining and incrementing variable.</li>
</ul>
</div>
<div class="section" id="spark-enhancements">
<h3>Spark Enhancements<a class="headerlink" href="#spark-enhancements" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4871">CDAP-4871</a> - Added capabilities to run PySpark programs in CDAP.</li>
</ul>
</div>
<div class="section" id="governance-and-security-enhancements">
<h3>Governance and Security Enhancements<a class="headerlink" href="#governance-and-security-enhancements" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12134">CDAP-12134</a> - Implemented the new authorization model for CDAP. The old authorization model is no longer supported.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12317">CDAP-12317</a> - Added a new configuration <code class="docutils literal notranslate"><span class="pre">security.authorization.extension.jar.path</span></code> in cdap-site.xml which can be used to add extra classpath and is avalible to cdap security extensions</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12100">CDAP-12100</a> - Removed automatic grant/revoke privileges on CDAP entity creation/deletion.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12367">CDAP-12367</a> - Added support for authorization on Kerberos principal for impersonation.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-11839">CDAP-11839</a> - Modified the authorization model so that read/write on an entity will not depend on its parent.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12135">CDAP-12135</a> - Deprecated <code class="docutils literal notranslate"><span class="pre">createFilter()</span></code> and added a new <code class="docutils literal notranslate"><span class="pre">isVisible</span></code> API in AuthorzationEnforcer. Deprecated grant/revoke APIs for EntityId and added new one for Authorizable which support wildcard privileges</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12283">CDAP-12283</a> - Removed version for artifacts for authorization policy to be consistent with applications. From 4.3 onwards CDAP does not support policies on artifact/application version.</li>
</ul>
</div>
<div class="section" id="id34">
<h3>Other New Features<a class="headerlink" href="#id34" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-11940">CDAP-11940</a> - Added a wizard to allow configuring and deploying microservices in UI.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-6329">CDAP-6329</a> - Enabled GC logging for CDAP services.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-11448">CDAP-11448</a> - Added support for HDInsight 3.6.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4874">CDAP-4874</a> - CSD now performs a version compatibility check with the active CDAP Parcel</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12348">CDAP-12348</a> - Added live migration of metrics tables from pre 4.3 tables to 4.3 salted metrics tables.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12017">CDAP-12017</a> - Added capability to salt the row key of the metrics tables so that writes are evenly distributed and there is no region hot spotting</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12068">CDAP-12068</a> - Added a REST API to check the status of metrics processor. We can view the topic level processing stats using this endpoint.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12070">CDAP-12070</a> - Added option to disable/enable metrics for a program through runtime arguments or preferences. This feature can also be used system wide by enabling/disabling metrics in cdap-site.xml</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12290">CDAP-12290</a> - Added global “CDAP” config to enable/disable metrics emission from user programs.By default metrics is enabled.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-1952">CDAP-1952</a> - DatasetOutputCommiter’s methods are now executed in the MapReduce ApplicationMaster, within OutputCommitter’s commitJob/abortJob methods. The MapReduceContext.addOutput(Output.of(String, OutputFormatProvider)) API can no longer be used to add OutputFormatProviders that also implement the DatasetOutputCommitter interface.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12084">CDAP-12084</a> - Allow appending to (or overwriting) a PartitionedFileSet’s partitions when using DynamicPartitioner APIs. Introduced a PartitionedFileSet.setMetadata API which now allows modifying partitions’ metadata.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12085">CDAP-12085</a> - Exposed a programmatic API to leverage Hive’s functionality to concatenate a partition of a PartitionedFileSet.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12378">CDAP-12378</a> - Workflow now allows adding configurable conditions with the lifecycle methods.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-8629">CDAP-8629</a> - Allow programs to have concurrent runs in integration test cases.</li>
</ul>
</div>
<div class="section" id="id35">
<h3>Bug Fixes<a class="headerlink" href="#id35" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12103">CDAP-12103</a> - Removed deprecated cdap-etl-realtime artifact.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12123">CDAP-12123</a> - Removed deprecated deprecated cdap-etl-batch jar from packaging.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-9150">CDAP-9150</a> - Allowed user to override the InputFormat class and OutputFormat class of a FileSet at runtime.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12285">CDAP-12285</a> - Fixed an issue with the order of HBase compatibility libraries in the class path.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-9125">CDAP-9125</a> - Fixed an issue where CDAP Sentry Integration did not rely on every user having their own individual group.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-11095">CDAP-11095</a> - Added support for a description field in a pipeline config that will be used as the application’s description if set.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12020">CDAP-12020</a> - Reuse network connections for TMS client.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12143">CDAP-12143</a> - Removes the existing hierarchal authorization model from CDAP</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12226">CDAP-12226</a> - Added an optional delimiter property to the HDFS sink to allow users to configure the delimiter used to separate record fields.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12298">CDAP-12298</a> - Individual system service status API no longer has to go through CDAP master.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-9953">CDAP-9953</a> - Removed dataset usage in the Hive source and sink, which allows it to work in Spark and fixes a race condition that could cause pipelines to fail with a transaction conflict exception.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-10228">CDAP-10228</a> - Sinks in streaming pipelines no longer have their <code class="docutils literal notranslate"><span class="pre">prepareRun</span></code> and <code class="docutils literal notranslate"><span class="pre">onFinish</span></code> methods called if the RDD for that batch is empty</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-11704">CDAP-11704</a> - Fixed CDAP to work with and publish to YARN Timeline Server in a secure environment.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-11783">CDAP-11783</a> - HBaseDDLExecutor implementation is now localized to the containers without adding it in the container classpath.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-11800">CDAP-11800</a> - Fixed a bug that the stream client gave wrong error message when the authorization check failed for stream read.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-11880">CDAP-11880</a> - Fixed a bug that caused pipelines and other programs to not create datasets at runtime with correct impersonated user.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-11944">CDAP-11944</a> - Removed non-configurable properties from CSD/Ambari</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-11948">CDAP-11948</a> - Fixed a bug where committed data could be removed during HBase table flush or compaction.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-11955">CDAP-11955</a> - Fixed a bug where sometimes wrong user was used in explore, which resulted in the failure of deleting namespace.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12054">CDAP-12054</a> - Fixed PartitionedFileSet to work with CombineFileInputFormat, as input to a batch job.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12122">CDAP-12122</a> - Fixed a bug in the pipeline planner that caused some pipelines to fail to deploy with a NoSuchElementException</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12125">CDAP-12125</a> - Fixed a bug in MapReduce pipeline timing metrics, where time for a stage could include time spent in other stages.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12130">CDAP-12130</a> - Fixed an issue that was causing send-to-directive to fail on derived columns in Data Prep.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12161">CDAP-12161</a> - Fixed a bug in StructuredRecord where a union of null and at least two other types could not be set to a null value.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12170">CDAP-12170</a> - Fixed a bug where committed files of a PartitionedFileSet could be removed during transaction rollback in the case PartitionOutput#addPartition was called for a partition that already existed. With this fix, PartitionedFileSet#getPartitionOutput should now only be called within a transaction.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12193">CDAP-12193</a> - Fixed a bug in some MapReduce pipelines that could cause duplicate reads if sources are not properly merged into the same MapReduce.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12199">CDAP-12199</a> - Fixed a bug that made local datasets inaccessible in a Workflow’s initialize and destroy methods.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12253">CDAP-12253</a> - Fixed a bug where the file batch source was always using a default schema instead of the actual output schema.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12269">CDAP-12269</a> - Fixed a bug that prevented pipelines from being published when plugin artifact versions were not specified</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12284">CDAP-12284</a> - Fixed a packaging bug that caused debian packages to include the wrong cdap-data-pipeline and cdap-data-streams artifacts for spark2.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12351">CDAP-12351</a> - Fixes an issue where truncating a file set did not preserve its base directory’s ownership and permissions.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12360">CDAP-12360</a> - Fixed an issue where certain excessive logging could cause a deadlock in CDAP master.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-12371">CDAP-12371</a> - In order to execute Hive queries using MR execution engine in CM 5.12 cluster, the ‘yarn.app.mapreduce.am.staging-dir’ property needs to be set to ‘/user’ in the YARN Configuration Safety Value in Cloudera Manager.</li>
</ul>
</div>
</div>
<div class="section" id="release-4-2-0">
<h2><a class="reference external" href="http://docs.cask.co/cdap/4.2.0/index.html">Release 4.2.0</a><a class="headerlink" href="#release-4-2-0" title="Permalink to this headline">🔗</a></h2>
<div class="section" id="id36">
<h3>Summary<a class="headerlink" href="#id36" title="Permalink to this headline">🔗</a></h3>
<p>1. <strong>Spark Enhancements:</strong> Added suppport for Apache Spark 2.x. Users have an option to configure CDAP to use Spark 1.x
or Spark 2.x on their cluster. Also added capability to run interactive Spark code within CDAP.</p>
<p>2. <strong>Enhanced 数据预处理:</strong> Added capabilities in data preparation to connect to the File System (Local and
HDFS) and relational databases, browse and select their existing data, and import into 数据预处理 for cleansing,
preparing and transforming.</p>
<p>3. <strong>Event Driven Schedules:</strong> Added capabilities to start CDAP programs based on data availability of partitions of
data in HDFS and pose run contraints to intelligently orchestrate CDAP Workflows.</p>
</div>
<div class="section" id="id37">
<h3>New Features<a class="headerlink" href="#id37" title="Permalink to this headline">🔗</a></h3>
</div>
<div class="section" id="id38">
<h3>Spark Enhancements<a class="headerlink" href="#id38" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7875">CDAP-7875</a> - Added support for Spark 2.x. In environments where multiple Spark versions exist, CDAP must be configured to use one or the other</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-11409">CDAP-11409</a>- Enable capabilities to run interactive Spark code within CDAP</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-11410">CDAP-11410</a> - Added capabilities to run arbitrary Spark code in CDAP Pipelines</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-11411">CDAP-11411</a> - Enhancements to speed up launching Spark programs</li>
</ul>
</div>
<div class="section" id="enhanced">
<h3>Enhanced 数据预处理<a class="headerlink" href="#enhanced" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-9290">CDAP-9290</a> - Adds File System Browser Component to browse Local and HDFS File System from 数据预处理</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-9517">CDAP-9517</a> - Adds Data Quality information to 数据预处理 table. Currently, it shows the completeness of each column</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-9524">CDAP-9524</a> - Added point-and-click interactions for applying directives such as parsing, splitting, find and replace, filling null or empty rows, copying and deleting columns in 数据预处理. They can be invoked by using the dropdown menu for each column</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-11333">CDAP-11333</a> - Added point-and-click interaction for cleansing column names</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-11334">CDAP-11334</a> - Added a point-and-click interaction to set all column names in 数据预处理</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-11424">CDAP-11424</a> - Added the ability to ingest data one-tim from 数据预处理 to a CDAP Dataset</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-9556">CDAP-9556</a> -  Added macro support for 数据预处理 directives</li>
</ul>
</div>
<div class="section" id="event-driven-schedules">
<h3>Event Driven Schedules<a class="headerlink" href="#event-driven-schedules" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7593">CDAP-7593</a> - Introduces a new, event-driven scheduling system that can start programs based on data availability in HDFS partitions</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-11338">CDAP-11338</a> - Allow users to configure constraints for schedules, such as duration since last run and allowed time range for program execution</li>
</ul>
</div>
<div class="section" id="id39">
<h3>Other New Features<a class="headerlink" href="#id39" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-11498">CDAP-11498</a> - Added capability for CDAP Services to dynamically list available artifacts and dynamically load artifacts</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7873">CDAP-7873</a> - Added support for EMR 5.0 - 5.3</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-11486">CDAP-11486</a> - Added the ability for 数据预处理 to handle byte arrays of data for processing binary data</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-11422">CDAP-11422</a> - Added an API to Spark Streaming sources to provide number of streams being used by a streaming source</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-11681">CDAP-11681</a> - Users can now upload, view, and use plugins of type ‘sparksink’ in Studio.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-8668">CDAP-8668</a> - Modified the log viewer to only show ERROR, WARN, and INFO levels of logs by default, instead of all logs as previously</li>
</ul>
</div>
<div class="section" id="id40">
<h3>Bug fixes<a class="headerlink" href="#id40" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-8289">CDAP-8289</a> - Fix a bug where the log level was always set to INFO at the root logger</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7727">CDAP-7727</a> - Fix a bug where extra characters after an artifact version range were being ignored instead of being recognized as invalid</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7884">CDAP-7884</a> - Fixed a bug where users could not read from real Datasets while previewing CDAP Pipelines</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-9422">CDAP-9422</a> - Fixed a bug that prevented users from adding extra classpath to Apache Spark drivers and executors</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-9456">CDAP-9456</a> - Fixed a bug where impersonated workflow was not creating local datasets with the correct impersonated user</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-11417">CDAP-11417</a> - Fixed a bug in Parquet and Avro File sinks that would cause them to fail if they received ByteBuffers instead of byte arrays.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-11558">CDAP-11558</a> - Fixed a bug where writes could only succeed in one MongoDB sink even when multiple MongoDB sinks were present in a pipeline</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-11577">CDAP-11577</a> - Fixed a thread leakage bug in Spark (SPARK-20935) after Spark Streaming program completed</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-11588">CDAP-11588</a> - Fixed a bug in TMS where fetching from the payload table raised an exception if the fetch had an empty result</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-11643">CDAP-11643</a> - Fixed a bug in the Purchase example that could cause purchases to overwrite each other</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-11651">CDAP-11651</a> - Fixed a bug that prevented from using logback.xml in Apache Spark Streaming programs.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-9284">CDAP-9284</a> -  Fixed an issue where pipeline metrics were not showing up in pipelines with a large number of nodes</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-11795">CDAP-11795</a> - Fixed an issue with retrieving workflow state if it contained an exception without a message</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-11445">CDAP-11445</a> - Fixed an issue with the CDAP Ambari service definition where the “cdap” headless user was not unique to the cluster</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4887">CDAP-4887</a> - Fixed the CDAP Upgrade tool to not fail when encountering a non-CDAP table that follows the CDAP naming convention</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-5067">CDAP-5067</a> - Fixed an issue where the driver process of a CDAP Workflow was getting restarted when it ran out of memory, causing the Workflow to be executed again from the start node</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7429">CDAP-7429</a> - Fixed an issue with the detection of Apache Spark on HDP 2.5 and above, which caused excess noise on the console</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-8888">CDAP-8888</a> - Fixed an issue with the YARN container allocation logic so that the correct container size is used.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-8911">CDAP-8911</a> - Fixed the stream container to terminate cleanly and cleaned up the CDAP Master’s Apache Twill JAR files after master shutdown</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-8918">CDAP-8918</a> - Fixed an issue where redeployment of an application with a deleted schedule would fail</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-8961">CDAP-8961</a> - Fixed warnings about /opt/cdap/master/artifacts not being a directory in unit tests</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-9026">CDAP-9026</a> - Fixed an issue due to which CDAP entity roles were not cleanup when the entity was deleted</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-9378">CDAP-9378</a> - Fixed an issue where cdap-security.xml was not written under Ambari unless security.enabled in cdap-site.xml was set to true</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-10475">CDAP-10475</a> - Fixed the Azure Blob Store source to work with Avro and Parquet formats</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-11384">CDAP-11384</a> - Fixed the Azure Blob Store source to work with CDAP FileSets</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-11557">CDAP-11557</a> - Fixed the “value is” filter in the 数据预处理 UI</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-11815">CDAP-11815</a> - Fixed impersonation while upgrading datasets in the Upgrade tool</li>
</ul>
</div>
<div class="section" id="deprecations">
<h3>Deprecations<a class="headerlink" href="#deprecations" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-8327">CDAP-8327</a> - Add property “metrics.processor.queue.size” with default value 20000 to limit the maximum size of a queue where metrics processor temporarily stores newly fetched metrics in memory before persisting them. Added property “metrics.processor.max.delay.ms” with default value 3000 milliseconds to specify the maximum delay allowed between the latest metrics timestamp and the time when it is processed. The larger this property is, Metrics Processor gets to sleep more often between fetching each batch of metrics but the delay between metrics emission and processing also increases. Deprecated the property “metrics.messaging.fetcher.limit”</li>
</ul>
</div>
</div>
<div class="section" id="release-4-1-1">
<h2><a class="reference external" href="http://docs.cask.co/cdap/4.1.1/index.html">Release 4.1.1</a><a class="headerlink" href="#release-4-1-1" title="Permalink to this headline">🔗</a></h2>
<div class="section" id="id41">
<h3>Summary<a class="headerlink" href="#id41" title="Permalink to this headline">🔗</a></h3>
<ol class="arabic simple">
<li><strong>数据预处理:</strong> Point-and-click interactions and integration with the rest of CDAP
including—but not limited to—namespaces, security, and pipelines.</li>
<li><strong>Upgrade:</strong> Significant reduction in downtime during CDAP upgrades, by removing some data
migration and doing required migration in the background after CDAP starts up.</li>
<li><strong>Pipeline Previews</strong>: Added logs, better error messaging, ability to read from existing
datasets, and a better stop experience.</li>
<li><strong>Logs</strong>: Added a condensed view of logs for CDAP pipelines and programs that does not
include logs emitted by the CDAP platform and libraries. The condensed view only
contains lifecycle logs, logs emitted by the program or pipeline, and errors.</li>
<li><strong>Schedules:</strong> Added the ability to update schedules without redeploying the application.</li>
</ol>
</div>
<div class="section" id="id42">
<h3>New Features<a class="headerlink" href="#id42" title="Permalink to this headline">🔗</a></h3>
<div class="section" id="id43">
<h4>数据预处理<a class="headerlink" href="#id43" title="Permalink to this headline">🔗</a></h4>
<ul class="simple">
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-9235">CDAP-9235</a> - Users can now interact with and manage multiple workspaces in
数据预处理.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/WRANGLER-77">WRANGLER-77</a> - Added point-and-click interactions for applying directives
such as parsing, splitting, find and replace, filling null or empty rows, copying and
deleting columns in 数据预处理. They can be invoked by using the dropdown menu for
each column.</li>
</ul>
</div>
<div class="section" id="logs">
<h4>Logs<a class="headerlink" href="#logs" title="Permalink to this headline">🔗</a></h4>
<ul class="simple">
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-9117">CDAP-9117</a> - Added option to the log viewer to only show “user” condensed logs.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-1316">HYDRATOR-1316</a> - Logs for previews of CDAP pipelines are now available in
the CDAP UI via the <em>Logs</em> button in Preview mode.</li>
</ul>
</div>
<div class="section" id="schedules">
<h4>Schedules<a class="headerlink" href="#schedules" title="Permalink to this headline">🔗</a></h4>
<ul class="simple">
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-8902">CDAP-8902</a> - Added support for adding, deleting, updating, and retrieving
workflow schedules.</li>
</ul>
</div>
<div class="section" id="id44">
<h4>Other New Features<a class="headerlink" href="#id44" title="Permalink to this headline">🔗</a></h4>
<ul>
<li><p class="first"><a class="reference external" href="https://issues.cask.co/browse/CDAP-8872">CDAP-8872</a> - Upgraded Apache Tephra dependency to the 0.11.0-incubating
version.</p>
</li>
<li><p class="first"><a class="reference external" href="https://issues.cask.co/browse/CDAP-9141">CDAP-9141</a>, <a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-1453">HYDRATOR-1453</a> - Users can now deploy CDAP
pipelines with a single action plugin. This feature can be used to run external Apache
Spark programs as CDAP pipelines.</p>
<p>Added a <em>sparkprogram</em> plugin type that can be used to run arbitrary Spark code at the
beginning or end of a pipeline. An external Spark program can be added by clicking the
“plus” (“+”) button in the CDAP UI, choosing <em>Library</em>, and specifying <em>sparkprogram</em> as
the type. It is then available as an Action plugin in the CDAP Studio.</p>
</li>
<li><p class="first"><a class="reference external" href="https://issues.cask.co/browse/CDAP-9250">CDAP-9250</a> - Added support for HDP 2.6.</p>
</li>
<li><p class="first"><a class="reference external" href="https://issues.cask.co/browse/CDAP-9281">CDAP-9281</a> - Added support for CDH 5.11.0.</p>
</li>
<li><p class="first"><a class="reference external" href="https://issues.cask.co/browse/CDAP-9311">CDAP-9311</a> - Added support that allows plugin developers to integrate with
CDAP services by exposing CDAP service discovery capabilities in the plugin context.</p>
</li>
</ul>
</div>
</div>
<div class="section" id="id45">
<h3>Improvements<a class="headerlink" href="#id45" title="Permalink to this headline">🔗</a></h3>
<div class="section" id="upgrade">
<h4>Upgrade<a class="headerlink" href="#upgrade" title="Permalink to this headline">🔗</a></h4>
<ul class="simple">
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-9278">CDAP-9278</a> - Added the running of HBase coprocessor upgrades concurrently
on CDAP Datasets.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-9282">CDAP-9282</a>, <a class="reference external" href="https://issues.cask.co/browse/CDAP-9283">CDAP-9283</a> - Improved the CDAP upgrade process to
minimize the downtime needed to upgrade, by performing data migration in the background.</li>
</ul>
</div>
<div class="section" id="pipeline-previews">
<h4>Pipeline Previews<a class="headerlink" href="#pipeline-previews" title="Permalink to this headline">🔗</a></h4>
<ul class="simple">
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-9017">CDAP-9017</a> - Simplified the status, next runtime of pipelines, total number
of running pipelines, and drafts in the pipeline list view UI.</li>
</ul>
</div>
<div class="section" id="id46">
<h4>Schedules<a class="headerlink" href="#id46" title="Permalink to this headline">🔗</a></h4>
<ul class="simple">
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-8942">CDAP-8942</a> - Allow administrators to enable or disable updating schedules
using the property “app.deploy.update.schedules” in cdap-site.xml. Users can override this
to enable or disable updating schedules during deployment of an application using the same
property specified in the configuration of the application.</li>
</ul>
</div>
<div class="section" id="other-improvements">
<h4>Other Improvements<a class="headerlink" href="#other-improvements" title="Permalink to this headline">🔗</a></h4>
<ul class="simple">
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7731">CDAP-7731</a> - Added fetch size and transaction flush interval configurations
to the Kafka Consumer Flowlet.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-8430">CDAP-8430</a> - Users can now see a contextual message with appropriate
call(s) to action when no entities are found on the Overview page.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-8990">CDAP-8990</a> - Added new configurations to control the YARN application
master container memory size, maximum heap memory size, and maximum non-heap memory size:
<code class="docutils literal notranslate"><span class="pre">twill.java.heap.memory.ratio</span></code>, <code class="docutils literal notranslate"><span class="pre">twill.yarn.am.memory.mb</span></code>, and
<code class="docutils literal notranslate"><span class="pre">twill.yarn.am.reserved.memory.mb</span></code>.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-9003">CDAP-9003</a> - Increased the default memory allocation for the CDAP Explore service
container to 2048MB.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-9027">CDAP-9027</a> - Users can now grant and revoke privileges for UNIX groups and
users when using Apache Sentry as the authorization extension for CDAP.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-9077">CDAP-9077</a> - Added a “cdap apply-pack [pack]” command to the “cdap” script
that allows for upgrading of individual CDAP components.</li>
</ul>
</div>
</div>
<div class="section" id="id47">
<h3>Bug Fixes<a class="headerlink" href="#id47" title="Permalink to this headline">🔗</a></h3>
<div class="section" id="id48">
<h4>Upgrade<a class="headerlink" href="#id48" title="Permalink to this headline">🔗</a></h4>
<ul class="simple">
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-9185">CDAP-9185</a> - Fixed an issue with the pipeline upgrade tool that caused it
to skip CDAP 4.0.x pipelines.</li>
</ul>
</div>
<div class="section" id="id49">
<h4>Pipeline Previews<a class="headerlink" href="#id49" title="Permalink to this headline">🔗</a></h4>
<ul class="simple">
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7884">CDAP-7884</a> - Fixed a bug that preview cannot read from datasets in real
space.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-8013">CDAP-8013</a> - When previewing a pipeline in the CDAP Studio, disabled all
writes to sinks. Incoming data to sinks can be viewed in the preview tab of the sink, but
is not written to the sink.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-9333">CDAP-9333</a> - Fixed an issue where preview of CDAP pipelines did not show
data for successful stages if a particular stage failed.</li>
</ul>
</div>
<div class="section" id="id50">
<h4>Logs<a class="headerlink" href="#id50" title="Permalink to this headline">🔗</a></h4>
<ul class="simple">
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7138">CDAP-7138</a> - Fixed a problem that caused duplicate logs to show up for a
running pipeline.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-9248">CDAP-9248</a> - Fixed bug where the “Total Messages/Errors/Warnings” at the
top of logviewer was showing incorrect values.</li>
</ul>
</div>
<div class="section" id="id51">
<h4>Schedules<a class="headerlink" href="#id51" title="Permalink to this headline">🔗</a></h4>
<ul class="simple">
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-8918">CDAP-8918</a> - Fixed an issue where redeployment of an application with a
deleted schedule would fail.</li>
</ul>
</div>
<div class="section" id="other-bug-fixes">
<h4>Other Bug Fixes<a class="headerlink" href="#other-bug-fixes" title="Permalink to this headline">🔗</a></h4>
<ul class="simple">
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4213">CDAP-4213</a> - Removed the requirement of being an admin to run the CDAP
startup script for Windows.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-5715">CDAP-5715</a> - Made Plugin Endpoint invocation more robust. If a plugin’s
parent can’t instantiate the plugin necessary for invoking, CDAP will attempt with other
parents of the plugin and try to instantiate using them before retuning error.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-6348">CDAP-6348</a> - Fixed an issue with namespace deletion which caused CDAP
Application test cases to fail in a Windows environment.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-8862">CDAP-8862</a> - Fix an issue with losing a few metrics when a container is
shutdown.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-8888">CDAP-8888</a> - Fixed an issue with the YARN container allocation logic so
that the correct container size is used.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-8913">CDAP-8913</a> - Improved the serializability of Tables and IndexedTables when
used in Spark programs.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-8945">CDAP-8945</a> - Moved the “add plugin” behavior from a plugin’s left panel to
an “Add Entity” button in the CDAP Studio UI.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-8950">CDAP-8950</a> - Fixed an issue in the CDAP UI where navigating from a stream
card to an overview and then to a detail page made the detail page show a spinner icon
indefinitely.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-8980">CDAP-8980</a>, <a class="reference external" href="https://issues.cask.co/browse/CDAP-9314">CDAP-9314</a> - Fixed an issue with the Spark program
runtime so that the Kryo serializer can be used.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-9005">CDAP-9005</a> - Fixed an issue where the HBase Queue Debugging Tool failed
when authorization was enabled.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-9029">CDAP-9029</a>, <a class="reference external" href="https://issues.cask.co/browse/CDAP-9035">CDAP-9035</a> - Fixed an issue where users could not
grant and revoke privileges for UNIX groups and users when using Apache Sentry as the
authorization extension for CDAP.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-9046">CDAP-9046</a> - Fixed an issue where revoking privileges from a role caused
the privilege to be revoked from all roles.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-9086">CDAP-9086</a> - Fixed an issue with the Window plugin so that it propagates
schema properly.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-9087">CDAP-9087</a> - Fixed the Overview panel in home page of the CDAP UI to handle
unknown entities appropriately.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-9114">CDAP-9114</a> - Added the retrying of local dataset operations when a failure
happens.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-9142">CDAP-9142</a> - Fixed an issue with the binary format in the Kafka streaming
source that prevented pipeline deployment.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-9160">CDAP-9160</a> - Fixed an issue that caused YARN containers to be killed due to
excessive memory usage when impersonation is enabled.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-9216">CDAP-9216</a> - Fixed bug where navigation links were referencing default
namespace instead of the current namespace.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-703">HYDRATOR-703</a> - Improved error messages for the ‘Get Schema’ functionality
of Database plugins in CDAP Pipelines.</li>
</ul>
</div>
</div>
<div class="section" id="id52">
<h3>Known Issues<a class="headerlink" href="#id52" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-9151">CDAP-9151</a> - The CDAP CLI commands for getting and setting preferences
introduced in CDAP 4.1.0 (such as <code class="docutils literal notranslate"><span class="pre">set</span> <span class="pre">app</span> <span class="pre">preferences</span> <span class="pre">&lt;app-id&gt;</span> <span class="pre">&lt;preferences&gt;</span></code>) are not
working correctly. Use the previous commands (marked as deprecated), such as <code class="docutils literal notranslate"><span class="pre">set</span>
<span class="pre">preferences</span> <span class="pre">app</span> <span class="pre">&lt;runtime-args&gt;</span> <span class="pre">&lt;app-id&gt;</span></code>, as a workaround.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-9388">CDAP-9388</a> - When creating a stream and uploading data from the wizard in
the CDAP resource center, the metrics on the cards in the overview do not show appropriate
numbers. It will just show zero for the number of events and the bytes.</li>
</ul>
</div>
<div class="section" id="api-changes">
<h3>API Changes<a class="headerlink" href="#api-changes" title="Permalink to this headline">🔗</a></h3>
<div class="section" id="id53">
<h4>Logs<a class="headerlink" href="#id53" title="Permalink to this headline">🔗</a></h4>
<ul class="simple">
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-9084">CDAP-9084</a> - The CDAP Logging APIs now return a 404 status code if the
entity (the run id) for which logs are requested does not exist.</li>
</ul>
</div>
</div>
</div>
<div class="section" id="release-4-1-0">
<h2><a class="reference external" href="http://docs.cask.co/cdap/4.1.0/index.html">Release 4.1.0</a><a class="headerlink" href="#release-4-1-0" title="Permalink to this headline">🔗</a></h2>
<div class="section" id="id54">
<h3>New Features<a class="headerlink" href="#id54" title="Permalink to this headline">🔗</a></h3>
<div class="section" id="secure-impersonation">
<h4>Secure Impersonation<a class="headerlink" href="#secure-impersonation" title="Permalink to this headline">🔗</a></h4>
<ul class="simple">
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-8110">CDAP-8110</a> - Added support for fine-grained impersonation at the CDAP
application, dataset, and stream level.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-8355">CDAP-8355</a> - Impersonated namespaces can be configured to disallow the
impersonation of the namespace owner when running CDAP Explore queries.</li>
</ul>
</div>
<div class="section" id="replication-and-resiliency">
<h4>Replication and Resiliency<a class="headerlink" href="#replication-and-resiliency" title="Permalink to this headline">🔗</a></h4>
<ul class="simple">
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7685">CDAP-7685</a> - Provided SPI hooks that users can implement for performing
HBase DDL operations.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-8025">CDAP-8025</a> - Added a tool to check a cluster’s replication status.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-8032">CDAP-8032</a> - CDAP context methods will now be retried according to a
program’s retry policy. These are governed by these properties:<ul>
<li><code class="docutils literal notranslate"><span class="pre">custom.action.retry.policy.base.delay.ms</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">custom.action.retry.policy.max.delay.ms</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">custom.action.retry.policy.max.retries</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">custom.action.retry.policy.max.time.secs</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">custom.action.retry.policy.type</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">flow.retry.policy.base.delay.ms</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">flow.retry.policy.max.delay.ms</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">flow.retry.policy.max.retries</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">flow.retry.policy.max.time.secs</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">flow.retry.policy.type</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">mapreduce.retry.policy.base.delay.ms</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">mapreduce.retry.policy.max.delay.ms</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">mapreduce.retry.policy.max.retries</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">mapreduce.retry.policy.max.time.secs</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">mapreduce.retry.policy.type</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">service.retry.policy.base.delay.ms</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">service.retry.policy.max.delay.ms</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">service.retry.policy.max.retries</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">service.retry.policy.max.time.secs</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">service.retry.policy.type</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">spark.retry.policy.base.delay.ms</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">spark.retry.policy.max.delay.ms</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">spark.retry.policy.max.retries</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">spark.retry.policy.max.time.secs</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">spark.retry.policy.type</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">system.log.process.retry.policy.base.delay.ms</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">system.log.process.retry.policy.max.retries</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">system.log.process.retry.policy.max.time.secs</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">system.log.process.retry.policy.type</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">system.metrics.retry.policy.base.delay.ms</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">system.metrics.retry.policy.max.retries</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">system.metrics.retry.policy.max.time.secs</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">system.metrics.retry.policy.type</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">worker.retry.policy.base.delay.ms</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">worker.retry.policy.max.delay.ms</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">worker.retry.policy.max.retries</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">worker.retry.policy.max.time.secs</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">worker.retry.policy.type</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">workflow.retry.policy.base.delay.ms</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">workflow.retry.policy.max.delay.ms</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">workflow.retry.policy.max.retries</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">workflow.retry.policy.max.time.secs</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">workflow.retry.policy.type</span></code></li>
</ul>
</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-8037">CDAP-8037</a> - Added a <code class="docutils literal notranslate"><span class="pre">master.manage.hbase.coprocessors</span></code> setting that can be
set to false on clusters where the CDAP coprocessors are deployed on every HBase node.</li>
</ul>
</div>
<div class="section" id="enhancements-to-the-new-cdap-ui">
<h4>Enhancements to the New CDAP UI<a class="headerlink" href="#enhancements-to-the-new-cdap-ui" title="Permalink to this headline">🔗</a></h4>
<ul class="simple">
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-8021">CDAP-8021</a> - Added the management of preferences at the application and
program levels.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-8198">CDAP-8198</a>, <a class="reference external" href="https://issues.cask.co/browse/CDAP-8199">CDAP-8199</a>, <a class="reference external" href="https://issues.cask.co/browse/CDAP-8214">CDAP-8214</a>,
<a class="reference external" href="https://issues.cask.co/browse/CDAP-8217">CDAP-8217</a> - The CDAP UI added dataset and stream detail and overviews.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-8203">CDAP-8203</a> - The CDAP UI added a “call-to-action” dialog after entity
creation, so users can easily perform actions on the newly-created entities.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-8282">CDAP-8282</a>, <a class="reference external" href="https://issues.cask.co/browse/CDAP-8376">CDAP-8376</a> - Users can now view events and logs of
programs in the new CDAP UI using the events and log view “fast-action” dialogs.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-8398">CDAP-8398</a> - Users now see on the CDAP UI homepage a “Just Added” section,
listing and highlighting any entities added in the last five minutes.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-208">HYDRATOR-208</a> - The CDAP UI added a duration timer to CDAP pipelines.</li>
</ul>
</div>
<div class="section" id="id55">
<h4>Logs<a class="headerlink" href="#id55" title="Permalink to this headline">🔗</a></h4>
<ul class="simple">
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7676">CDAP-7676</a>, <a class="reference external" href="https://issues.cask.co/browse/CDAP-9999">CDAP-9999</a> - Added a prototype implementation for a rolling HDFS log
appender.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7962">CDAP-7962</a> - Program context information, including namespace, program
name, and program type, are now available in the MDC property of each ILoggingEvent
emitted from a program container.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-8108">CDAP-8108</a> - Revised the CDAP Log Appender to use <a class="reference external" href="http://logback.qos.ch/">Logback</a>’s Appender interface.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-8231">CDAP-8231</a> - The log file cleaner thread will remove metadata and, for
successfully deleted metadata entries, it will delete the corresponding log files. The log
file cleaner thread will only remove the metadata entries for the old (pre-4.1.0) log
format.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-8261">CDAP-8261</a> - Logs collected by the CDAP Log Appender will be stored at a
common <code class="docutils literal notranslate"><span class="pre">&lt;cdap&gt;/logs</span></code> path, owned by the cdap user. For security, it is readable only by
the cdap user.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-8428">CDAP-8428</a> - Added additional metrics about the status of the log
framework: <code class="docutils literal notranslate"><span class="pre">log.process.min.delay</span></code> and <code class="docutils literal notranslate"><span class="pre">log.process.max.delay</span></code>.</li>
</ul>
</div>
<div class="section" id="new-cdap-pipeline-plugins">
<h4>New CDAP Pipeline Plugins<a class="headerlink" href="#new-cdap-pipeline-plugins" title="Permalink to this headline">🔗</a></h4>
<ul class="simple">
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-235">HYDRATOR-235</a> - The Kinesis Spark Streaming source plugin is available in
its own repository at <a class="reference external" href="https://github.com/hydrator/kinesis-spark-streaming-source">github.com/hydrator/kinesis-spark-streaming-source</a>.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-552">HYDRATOR-552</a> - Added a plugin for sampling data from a source, available
at <a class="reference external" href="https://github.com/hydrator/sampling-aggregator">github.com/hydrator/sampling-aggregator</a>.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-585">HYDRATOR-585</a> - The HTTP Sink plugin (for posting data from a pipeline to
an external endpoint) has been added at <a class="reference external" href="https://github.com/hydrator/http-sink">github.com/hydrator/http-sink</a>.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-954">HYDRATOR-954</a> - The Kinesis Source plugin now works in realtime pipelines.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-983">HYDRATOR-983</a> - Added a Feature Generator plugin for a pipeline builder.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-1049">HYDRATOR-1049</a> - Added a DynamoDb Sink as a plugin, available at
<a class="reference external" href="https://github.com/hydrator/dynamodb-sink">github.com/hydrator/dynamodb-sink</a>.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-1050">HYDRATOR-1050</a> - Added a DynamoDB Batch Source plugin, available at
<a class="reference external" href="https://github.com/hydrator/dynamodb-source">github.com/hydrator/dynamodb-source</a>.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-1073">HYDRATOR-1073</a> - Added a “Fail This Pipeline” sink plugin in a repo at
<a class="reference external" href="https://github.com/hydrator/failpipeline-sink">github.com/hydrator/failpipeline-sink</a>;
this is a sink where, if any records flow to the sink, the pipeline is marked as failed,
triggering any post-actions that might be scheduled.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-1074">HYDRATOR-1074</a> - Added a plugin for fetching data from an external HTTP
site and writing the response to HDFS, available at
<a class="reference external" href="https://github.com/hydrator/httptohdfs-action">github.com/hydrator/httptohdfs-action</a>.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-1172">HYDRATOR-1172</a> - Added a Realtime Stream Source plugin, available at
<a class="reference external" href="https://github.com/hydrator/realtime-stream-source">github.com/hydrator/realtime-stream-source</a>.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-1249">HYDRATOR-1249</a> - The Tokenizer plugin is now available in it own repository
at <a class="reference external" href="https://github.com/hydrator/tokenizer-analytics">github.com/hydrator/tokenizer-analytics</a>.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-1250">HYDRATOR-1250</a> - The NGramTransform plugin is now available in its own
repository at <a class="reference external" href="https://github.com/hydrator/ngram-analytics">github.com/hydrator/ngram-analytics</a>.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-1251">HYDRATOR-1251</a> - The DecisionTree Regression plugins are now available in
their own repository at <a class="reference external" href="https://github.com/hydrator/decision-tree-analytics">github.com/hydrator/decision-tree-analytics</a>.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-1252">HYDRATOR-1252</a> - The SkipGram Feature Generator plugin is now available in
its own repository at <a class="reference external" href="https://github.com/hydrator/skipgram-analytics">github.com/hydrator/skipgram-analytics</a>.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-1253">HYDRATOR-1253</a> - The Naive Bayes Analytics plugin is now available in its
own repository at <a class="reference external" href="https://github.com/hydrator/naive-bayes-analytics">github.com/hydrator/naive-bayes-analytics</a>.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-1254">HYDRATOR-1254</a> - The HashingTF Feature Generator plugin is now available in
its own repository at <a class="reference external" href="https://github.com/hydrator/hashing-tf-feature-generator">github.com/hydrator/hashing-tf-feature-generator</a>.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-1255">HYDRATOR-1255</a> - The LogisticRegression plugins are now available in their
own repository at <a class="reference external" href="https://github.com/hydrator/logistic-regression-analytics">github.com/hydrator/logistic-regression-analytics</a>.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-1323">HYDRATOR-1323</a> - Added a new ErrorTransform plugin-type that can be placed
after a pipeline stage to consume errors emitted by that stage.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-1398">HYDRATOR-1398</a> - Support added for Table datasets for lookups in plugins
and pipelines.</li>
</ul>
</div>
<div class="section" id="dataset-improvements">
<h4>Dataset Improvements<a class="headerlink" href="#dataset-improvements" title="Permalink to this headline">🔗</a></h4>
<ul class="simple">
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7596">CDAP-7596</a> - Added the ability to reuse an existing file system location
and Hive table when creating a partitioned file set.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7597">CDAP-7597</a> - Added configuring the CDAP Explore database and table name for
a dataset using dataset properties.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7683">CDAP-7683</a> - Added a tool that pre-builds and loads the HBase coprocessors
required by CDAP onto HDFS.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-8070">CDAP-8070</a> - Added control of group ownership and permissions through
dataset properties.</li>
</ul>
</div>
<div class="section" id="id56">
<h4>Other New Features<a class="headerlink" href="#id56" title="Permalink to this headline">🔗</a></h4>
<ul class="simple">
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4556">CDAP-4556</a> - CDAP now uses environment variables in the <code class="docutils literal notranslate"><span class="pre">spark-env.sh</span></code> and
properties in the <code class="docutils literal notranslate"><span class="pre">spark-defaults.conf</span></code> when launching Spark programs.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-5107">CDAP-5107</a> - Added an HTTP RESTful endpoint to retrieve a specific property
for a specific version of an artifact in the <code class="docutils literal notranslate"><span class="pre">system</span></code> scope.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-8122">CDAP-8122</a> - Made headers and the request/response bodies available in
audit logs for certain RESTful endpoints.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-8292">CDAP-8292</a> - Added support for CDH 5.10.0.</li>
</ul>
</div>
</div>
<div class="section" id="id57">
<h3>Improvements<a class="headerlink" href="#id57" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3383">CDAP-3383</a> - Enabled in CDAP invalid transaction list pruning, a new
feature introduced in Apache Tephra. This automates the pruning of the invalid transaction
list after data for the invalid transaction has been dropped.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-6046">CDAP-6046</a> - Added an easier, additional syntax for the CDAP CLI
<code class="docutils literal notranslate"><span class="pre">set/get/load/delete</span> <span class="pre">&lt;type&gt;</span> <span class="pre">preferences</span></code> commands, with the preferences at the end of the
syntax, such as <code class="docutils literal notranslate"><span class="pre">set</span> <span class="pre">workflow</span> <span class="pre">preferences</span> <span class="pre">MyApp.My.WF</span> <span class="pre">'a=b</span> <span class="pre">c=d'</span></code>.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7835">CDAP-7835</a> - The Metadata Service upgrades the metadata dataset to reduce
the time required by the upgrade tool during a CDAP upgrade.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-8019">CDAP-8019</a> - Added a configuration to control the timeout of CDAP Explore
operations: set <code class="docutils literal notranslate"><span class="pre">explore.http.timeout</span></code> in the <code class="docutils literal notranslate"><span class="pre">cdap-site.xml</span></code> file.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-8061">CDAP-8061</a> - Moved the Cask Market Path to the <code class="docutils literal notranslate"><span class="pre">cdap-defaults.xml</span></code> file.
Users can now configure the path to a private Cask Market using the configuration
setting <code class="docutils literal notranslate"><span class="pre">market.base.url</span></code>.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-8075">CDAP-8075</a> - The CDAP UI added one-step deploy wizards for the Cask Market.
Users can now deploy applications and plugins from the Cask Market with a single click,
instead of downloading them from the market and then uploading them.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-8152">CDAP-8152</a> - StreamingSource plugins now have access to the CDAP
SparkExecutionContext to read from datasets and streams.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-8183">CDAP-8183</a> - The CDAP UI now automatically retries loading the homepage
when the CDAP Server is not up and ready yet.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-8250">CDAP-8250</a> - Reduced non-informative stacktrace information in the log when
a connection to the CDAP Router is closed prematurely.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-8565">CDAP-8565</a> - Improved the master process stop procedure to support fast
failover when running with HA. Added a new kill command to force-kill CDAP processes.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-282">HYDRATOR-282</a> - Updated the CSVParser plugin to change “PDL” to “Pipe
Delimited” and “TDF” to “Tab Delimited”.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-577">HYDRATOR-577</a> - Changed the Table sink plugin to make using the
<code class="docutils literal notranslate"><span class="pre">schema.row.field</span></code> optional, which allows the <code class="docutils literal notranslate"><span class="pre">schema.row.field</span></code> to be used as a
column in the output.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-1006">HYDRATOR-1006</a> - Updated the Tokenizer plugin to be more forgiving when
parsing tokens by accepting regex with white spaces; the output schema now contains all
the fields that were in the input schema and not only the column that is being tokenized.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-1028">HYDRATOR-1028</a> - Changed the Data Generator configuration to be easier to
use; as the type parameter can only be one of “stream” or “table”, changed to using a
select widget to configure it.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-1144">HYDRATOR-1144</a> - Updated the use of “true/false” select boxes to be
consistent in their ordering.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-1149">HYDRATOR-1149</a> - Added the ability to read recursive directories to the
File source plugin.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-1162">HYDRATOR-1162</a> - Added logging to an error-dataset to the LogParser and
XMLMultiParser plugins.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-1177">HYDRATOR-1177</a> - Plugins can now retrieve the input and output schema of
their stage in their initialize methods.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/WRANGLER-3">WRANGLER-3</a> - The CDAP UI’s Wrangler modal dialog will give a warning when
you try to close or exit out of it without confirmation.</li>
</ul>
</div>
<div class="section" id="id58">
<h3>Bug Fixes<a class="headerlink" href="#id58" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2543">CDAP-2543</a> - Fixed an issue of a hanging application in the case that a
user program JAR is missing dependencies.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4739">CDAP-4739</a> - Fixed an issue to make artifact, datasets, logs, and
coprocessor JAR locations resilient to an HDFS Namenode HA upgrade.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-5717">CDAP-5717</a> - Fixed an issue with starting the CDAP CLI and the CDAP
Standalone when the on-disk path has a space in it.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-6690">CDAP-6690</a> - Fixed issues with the formatting of dataset instance
properties in the output of the CDAP CLI.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-6704">CDAP-6704</a> - Fixed issues with and clarified certain of the CDAP CLI help
text and its error messages.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7155">CDAP-7155</a> - Fixed a problem where the Dataset Service failed to start up
if authorization was enabled and the authorization plugin was slow to respond.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7228">CDAP-7228</a> - Empty and null metadata tags are now removed in the metadata
upgrade step of the CDAP Upgrade Tool.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7302">CDAP-7302</a> - Fixed an issue that caused the CDAP Master to die if HBase was
down when a follower became the leader.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7694">CDAP-7694</a> - Fixed an issue where the CDAP service scripts could cause a
terminal session to not echo characters.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7813">CDAP-7813</a> - The security policies for accessing entities have been changed
and the documentation updated to reflect these changes.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7911">CDAP-7911</a> - The error messages returned for bad requests to the metadata
search RESTful APIs have been improved.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7930">CDAP-7930</a> - Performing a metadata search now returns the correct total,
even if the offset is very large.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7935">CDAP-7935</a> - Fixed an issue with the CDAP Standalone not starting and
stopping correctly.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7991">CDAP-7991</a> - The Cask Market now shows only those entities that are valid
for the specific version of CDAP viewing them.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-8001">CDAP-8001</a> - Fixed an issue with the retrieving of logs when a namespace
was deleted and then recreated with same name.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-8041">CDAP-8041</a> - Fixed an issue where the CDAP Master process would hang during
a shutdown.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-8086">CDAP-8086</a> - Removed an obsolete Update Dataset Specifications step in the
CDAP Upgrade tool. This step was required only for upgrading from CDAP versions lower than
3.2 to CDAP version 3.2.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-8087">CDAP-8087</a> - Provided a workaround for Scala bug SI-6240
(<a class="reference external" href="https://issues.scala-lang.org/browse/SI-6240">issues.scala-lang.org/browse/SI-6240</a>) to allow concurrent execution of
Spark programs in CDAP Workflows.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-8088">CDAP-8088</a> - Fixed the CDAP UI pipeline detail view so that it can be
rendered in older browsers.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-8094">CDAP-8094</a> - Fixed an issue where the number of records processed during a
preview run of the realtime data pipeline was being incremented incorrectly.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-8133">CDAP-8133</a> - Fixed an issue with metadata searches with certain offsets
overflowing and returning an error.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-8180">CDAP-8180</a> - Fixed an issue with the CDAP Standalone not correctly warning
about the absence of Node.js.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-8229">CDAP-8229</a> - Fix the CDAP UpgradeTool to not rely on the existence of a
‘default’ namespace.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-8313">CDAP-8313</a> - Fixed an issue where system artifacts would continuously be
loaded if there was a partial JAR in the system artifacts directory.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-8342">CDAP-8342</a> - Fixed an issue where CDAP Explore operations from a program
container running as a user were impersonating the namespace owner. Now they impersonate
the respective program container users.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-8367">CDAP-8367</a> - Fixed issues with “Hive-on-Spark” on newer versions of CDH
failing to run Spark jobs due to permission and configuration errors.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-8442">CDAP-8442</a> - Fixed an issue in the CDAP UI where the “Stop Program” modal
dialog kept loading (showing a spinning wheel) even after the program had been stopped.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-8446">CDAP-8446</a> - Fixed an issue where the Transactional.run method could throw
the wrong exception if the transaction service was unavailable when it was finishing a
transaction.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-8509">CDAP-8509</a> - Fixed an issue in the Transactional Messaging System (TMS)
table upgrade, where the TMS table could be left in a disabled state if the upgrade tool
is run after an upgraded CDAP Master is started and then stopped.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-8544">CDAP-8544</a> - Lowered the RPC timeout and number of retries for the HBase
operations performed by CDAP Master services.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-8628">CDAP-8628</a> - Fixed an issue in the log saver and the metrics processor that
if an exception was thrown during the changing of the number of instances, a container JVM
process could be left running without performing any work.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-8634">CDAP-8634</a> - Corrected the Javadoc of the PluginConfig’s containsMacro()
method to reflect that it always returns false at runtime.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-8636">CDAP-8636</a> - Fixed an issue with Spark programs not working against CDH
5.8.4.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-8672">CDAP-8672</a> - Fixed the CDAP Router so that it does not log an error when it
cannot discover a service. Previously, the message was logged at the debug level.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-8687">CDAP-8687</a> - Fixed an issue where a user who attempts to create an existing
stream that was created by a different user received all the privileges and the original
user had their privileges revoked.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-8694">CDAP-8694</a> - Fixed an issue with properly-locating CDAP_HOME in Distributed
CDAP instances outside the default <code class="docutils literal notranslate"><span class="pre">/opt/cdap</span></code> directory.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-1085">HYDRATOR-1085</a> - Fixed an issue where the File Sink plugin was failing when
writing byte array records.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-1096">HYDRATOR-1096</a> - Fixed an issue with the macro substitution of a Table
dataset name.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-1158">HYDRATOR-1158</a> - Fixed an issue with the JSON parser failing if no data was
present for a nullable field.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-1212">HYDRATOR-1212</a> - Fixed an issue where runtime arguments were not being
passed correctly for the pipeline preview run in the CDAP UI.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-1219">HYDRATOR-1219</a> - Fixed an issue in the Wrangler transform with the handling
of escaped characters.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-1226">HYDRATOR-1226</a> - Fixed an issue where pipeline previews would not run in a
non-default namespace.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-1238">HYDRATOR-1238</a> - Fixed an issue where the RunTransform plugin was not
checking for null fields.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-1246">HYDRATOR-1246</a> - Fixed an issue with the DateTransform plugin and the
handling of null values.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-1377">HYDRATOR-1377</a> - Fixed an issue with the S3 source and sink plugins in the
CDAP Standalone.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/TRACKER-264">TRACKER-264</a> - Fixed an issue with the Data Dictionary’s validate API not
accepting CDAP-schema JSON.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/WRANGLER-12">WRANGLER-12</a> - Added to Wrangler an option to convert column names to be
schema-compatible.</li>
</ul>
</div>
<div class="section" id="id59">
<h3>Known Issues<a class="headerlink" href="#id59" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7770">CDAP-7770</a> - The current CDAP UI build process does not work on Microsoft Windows.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-8375">CDAP-8375</a> - Invalid Transaction Pruning does not work on a replicated
cluster. and needs to be disabled by setting the configuration parameter
<code class="docutils literal notranslate"><span class="pre">data.tx.prune.enable</span></code> to <code class="docutils literal notranslate"><span class="pre">false</span></code> in the <code class="docutils literal notranslate"><span class="pre">cdap-site.xml</span></code> file.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-8494">CDAP-8494</a> - If users navigate to the classic CDAP UI, they cannot come
back to the new CDAP UI if they click the browser back button.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-8531">CDAP-8531</a>, <a class="reference external" href="https://issues.cask.co/browse/CDAP-8659">CDAP-8659</a>, <a class="reference external" href="https://issues.cask.co/browse/CDAP-8791">CDAP-8791</a> - If the
property <code class="docutils literal notranslate"><span class="pre">hive.compute.query.using.stats</span></code> is <code class="docutils literal notranslate"><span class="pre">true</span></code> in HDP 2.5.x clusters, CDAP
Explore queries that trigger a MapReduce program can fail.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-8663">CDAP-8663</a> - If a user revokes a privilege on a namespace, the privilege on
all entities in that namespace are also revoked.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-8789">CDAP-8789</a> - On the CDAP UI, program logs show error logs correctly. When
switched to “Raw Logs”, the error logs are missing. (The same behavior is seen in the
classic CDAP UI.) CDAP CLI shows all logs correctly.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-8812">CDAP-8812</a> - Long plugin names don’t show up in the left sidebar of the
CDAP Studio when running on Microsoft Windows.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-8818">CDAP-8818</a> - Local datasets appear on the CDAP UI overview page even though
they are temporary datasets that should be filtered out.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-1389">HYDRATOR-1389</a> - On Windows, users of CDAP Studio must double-click plugin icons
in order for their node configuration panels to open.</li>
</ul>
</div>
<div class="section" id="id60">
<h3>API Changes<a class="headerlink" href="#id60" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-6642">CDAP-6642</a> - Attempting to delete a system artifact by specifying a user
namespace (that previously returned a 200, even though the artifact was not deleted) will
now return a 404, as that combination of system and user will never occur.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-8445">CDAP-8445</a> - The stream endpoint to enqueue messages now returns a 503
instead of a 500 if it failed because the dataset service was unavailable.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-8448">CDAP-8448</a> - In general, changed the HTTP RESTful endpoints to return a 503
instead of a 500 when the transaction service was unavailable.</li>
</ul>
<ul id="release-notes-cdap-8606">
<li><p class="first"><a class="reference external" href="https://issues.cask.co/browse/CDAP-8606">CDAP-8606</a> - Among other new properties added to CDAP, new log saver
properties have been added to CDAP, replacing the previous properties. As a consequence,
previous properties will no longer work. See the <a class="reference external" href="http://docs.cask.co/cdap/4.1.0/en/admin-manual/appendices/cdap-site.html">Appendix: cdap-site.xml</a> for
details on these properties.</p>
<p><strong>Old Properties</strong></p>
<ul class="simple">
<li><code class="docutils literal notranslate"><span class="pre">log.cleanup.max.num.files</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">log.cleanup.run.interval.mins</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">log.retention.duration.days</span></code></li>
</ul>
<p><strong>New Properties</strong></p>
<ul class="simple">
<li><code class="docutils literal notranslate"><span class="pre">custom.action.retry.policy.base.delay.ms</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">custom.action.retry.policy.max.delay.ms</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">custom.action.retry.policy.max.retries</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">custom.action.retry.policy.max.time.secs</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">custom.action.retry.policy.type</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">data.tx.prune.enable</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">data.tx.prune.plugins</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">data.tx.prune.state.table</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">data.tx.pruning.plugin.class</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">explore.http.timeout</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">flow.retry.policy.base.delay.ms</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">flow.retry.policy.max.delay.ms</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">flow.retry.policy.max.retries</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">flow.retry.policy.max.time.secs</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">flow.retry.policy.type</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">hbase.client.retries.number</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">hbase.rpc.timeout</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">log.pipeline.cdap.dir.permissions</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">log.pipeline.cdap.file.cleanup.batch.size</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">log.pipeline.cdap.file.cleanup.transaction.timeout</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">log.pipeline.cdap.file.max.lifetime.ms</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">log.pipeline.cdap.file.max.size.bytes</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">log.pipeline.cdap.file.permissions</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">log.pipeline.cdap.file.retention.duration.days</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">log.pipeline.cdap.file.sync.interval.bytes</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">log.process.pipeline.auto.buffer.ratio</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">log.process.pipeline.buffer.size</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">log.process.pipeline.checkpoint.interval.ms</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">log.process.pipeline.config.dir</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">log.process.pipeline.event.delay.ms</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">log.process.pipeline.kafka.fetch.size</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">log.process.pipeline.lib.dir</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">log.process.pipeline.logger.cache.expiration.ms</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">log.process.pipeline.logger.cache.size</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">log.publish.partition.key</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">mapreduce.retry.policy.base.delay.ms</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">mapreduce.retry.policy.max.delay.ms</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">mapreduce.retry.policy.max.retries</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">mapreduce.retry.policy.max.time.secs</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">mapreduce.retry.policy.type</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">market.base.url</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">master.manage.hbase.coprocessors</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">metrics.kafka.meta.table</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">metrics.kafka.topic.prefix</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">metrics.messaging.fetcher.limit</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">metrics.messaging.meta.table</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">metrics.messaging.topic.num</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">metrics.topic.prefix</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">router.audit.path.check.enabled</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">security.keytab.path</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">service.retry.policy.base.delay.ms</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">service.retry.policy.max.delay.ms</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">service.retry.policy.max.retries</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">service.retry.policy.max.time.secs</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">service.retry.policy.type</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">spark.retry.policy.base.delay.ms</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">spark.retry.policy.max.delay.ms</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">spark.retry.policy.max.retries</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">spark.retry.policy.max.time.secs</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">spark.retry.policy.type</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">system.log.process.retry.policy.base.delay.ms</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">system.log.process.retry.policy.max.retries</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">system.log.process.retry.policy.max.time.secs</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">system.log.process.retry.policy.type</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">system.metrics.retry.policy.base.delay.ms</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">system.metrics.retry.policy.max.retries</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">system.metrics.retry.policy.max.time.secs</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">system.metrics.retry.policy.type</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">twill.location.cache.dir</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">worker.retry.policy.base.delay.ms</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">worker.retry.policy.max.delay.ms</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">worker.retry.policy.max.retries</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">worker.retry.policy.max.time.secs</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">worker.retry.policy.type</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">workflow.retry.policy.base.delay.ms</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">workflow.retry.policy.max.delay.ms</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">workflow.retry.policy.max.retries</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">workflow.retry.policy.max.time.secs</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">workflow.retry.policy.type</span></code></li>
</ul>
</li>
</ul>
</div>
<div class="section" id="id61">
<h3>Deprecated and Removed Features<a class="headerlink" href="#id61" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li>See <a class="reference internal" href="#release-notes-cdap-8606"><span class="std std-ref">API Changes, CDAP-8606</span></a> above for removed properties.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-8753">CDAP-8753</a> - Deprecated the <code class="docutils literal notranslate"><span class="pre">waitForFinish()</span></code> method in the ProgramManager and
added the method <code class="docutils literal notranslate"><span class="pre">waitForRun()</span></code> to replace it which will wait for the actual run
records of the given status.</li>
</ul>
</div>
</div>
<div class="section" id="release-4-0-1">
<h2><a class="reference external" href="http://docs.cask.co/cdap/4.0.1/index.html">Release 4.0.1</a><a class="headerlink" href="#release-4-0-1" title="Permalink to this headline">🔗</a></h2>
<div class="section" id="improvement">
<h3>Improvement<a class="headerlink" href="#improvement" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-8047">CDAP-8047</a> - Added a step in the CDAP Upgrade Tool to disable TMS
(Transaction Messaging Service) message and payload tables. The TMS TwillRunnable will
update the coprocessors of those tables if required and enable the tables.</li>
</ul>
</div>
<div class="section" id="id62">
<h3>Bug Fixes<a class="headerlink" href="#id62" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7694">CDAP-7694</a> - Fixed an issue where the CDAP service scripts could cause a
terminal session to not echo characters.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7992">CDAP-7992</a> - The CDAP Security service under Standalone CDAP is no longer
forced to bind to localhost.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-8000">CDAP-8000</a> - To avoid transaction timeouts, log cleanup is now done in
configurable batches (controlled by the property log.cleanup.max.num.files) instead of a
single short transaction.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-8007">CDAP-8007</a> - Fixed a bug in the TMS (Transaction Messaging Service) message
and payload table coprocessors by changing the accessing of CDAP configuration and TMS
metadata tables from reading them inline to reading them in a separate thread.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-8023">CDAP-8023</a> - Changed the default CDAP UI port to 11011 to match the CDAP
4.0.0 release.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-8086">CDAP-8086</a> - Removed an obsolete Update Dataset Specifications step in the
CDAP Upgrade tool. This step was required only for upgrading from CDAP versions lower than
3.2 to CDAP Version 3.2.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-8087">CDAP-8087</a> - Provided a workaround for Scala bug SI-6240
(<a class="reference external" href="https://issues.scala-lang.org/browse/SI-6240">https://issues.scala-lang.org/browse/SI-6240</a>) to allow concurrent execution of Spark
programs in CDAP Workflows.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-8088">CDAP-8088</a> - Fixed the CDAP Hydrator detail view so that it can be rendered
in older browsers.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-8094">CDAP-8094</a> - Fixed an issue where the number of records processed during a
preview run of the realtime data pipeline was being incremented incorrectly.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-8126">CDAP-8126</a> - Fixed an issue with the flag used by the Node proxy to enable
SSL between the CDAP UI and CDAP Router.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-8137">CDAP-8137</a> - Fixed an issue with the CDAP CLI where execute commands may be
interpreted incorrectly.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-8148">CDAP-8148</a> - Fixed an issue in the template path used with the original
CDAP UI when rendering a dataset detailed view.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-8158">CDAP-8158</a> - Fixed issues with the Ambari UI “Quick Links” and alerts
definitions for SSL and non-default ports and the writing of the cdap-security.xml file
when configured under the CDAP Ambari Service.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-1212">HYDRATOR-1212</a> - Fixed an issue where runtime arguments were not being
passed for the preview run correctly in the CDAP UI.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-1226">HYDRATOR-1226</a> - Fixed an issue where previews would not run in a
non-default namespace.</li>
</ul>
</div>
</div>
<div class="section" id="release-4-0-0">
<h2><a class="reference external" href="http://docs.cask.co/cdap/4.0.0/index.html">Release 4.0.0</a><a class="headerlink" href="#release-4-0-0" title="Permalink to this headline">🔗</a></h2>
<div class="section" id="id63">
<h3>New Features<a class="headerlink" href="#id63" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li>Cask Market<ul>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7203">CDAP-7203</a> - Adds Cask Market: Cask’s <em>Big Data</em> app store, providing an
ecosystem of pre-built Hadoop solutions, re-usable templates, and plugins. Within CDAP,
users can access the market and create Hadoop solutions or <em>Big Data</em> applications with
easy-to-use guided wizards.</li>
</ul>
</li>
<li>Cask Wrangler<ul>
<li><a class="reference external" href="https://issues.cask.co/browse/WRANGLER-2">WRANGLER-2</a> - Added Cask Wrangler: a new CDAP extension for interactive
data preparation.</li>
</ul>
</li>
<li>CDAP Transactional Messaging System<ul>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7211">CDAP-7211</a> - Adds a transactional messaging system that is used for
reliable communication of messages between components. In CDAP 4.0.0, the transactional
messaging system replaces Kafka for publishing and subscribing audit logs that is used
within CDAP for computing data lineage.</li>
</ul>
</li>
<li>Operational Statistics<ul>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7670">CDAP-7670</a> - Added a pluggable extension to retrieve operational statistics
in CDAP. Provided extensions for operational stats from YARN, HDFS, HBase, and CDAP.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7703">CDAP-7703</a> - Added reporting operational statistics for YARN. They can be
retrieved using JMX with the domain name <code class="docutils literal notranslate"><span class="pre">io.cdap.cdap.operations</span></code> and the property
<code class="docutils literal notranslate"><span class="pre">name</span></code> set to <code class="docutils literal notranslate"><span class="pre">yarn</span></code>.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7704">CDAP-7704</a> - Added reporting operational statistics for HBase. They can be
retrieved using JMX with the domain name <code class="docutils literal notranslate"><span class="pre">io.cdap.cdap.operations</span></code> and the property
<code class="docutils literal notranslate"><span class="pre">name</span></code> set to <code class="docutils literal notranslate"><span class="pre">hbase</span></code> as well as through the CDAP UI Administration page.</li>
</ul>
</li>
<li>Dynamic Log Level<ul>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-5479">CDAP-5479</a> - Allow updating or resetting of log levels for program types
worker, flow, and service dynamically using REST endpoints.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7214">CDAP-7214</a> - Allow setting the log levels for all program types through
runtime arguments or preferences.</li>
</ul>
</li>
<li>New Versions of Distributions Supported<ul>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-6938">CDAP-6938</a> - Added support for Amazon EMR 4.6.0+ installation of CDAP via a
bootstrap action script.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7249">CDAP-7249</a> - Added support for HDInsights 3.5.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7291">CDAP-7291</a> - Added support for CDH 5.9.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7901">CDAP-7901</a> - Added support for HDP 2.5.</li>
</ul>
</li>
<li>New Hydrator Plugins Added<ul>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-504">HYDRATOR-504</a> - Added to the Hydrator plugins a Tokenizer Spark compute
plugin.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-512">HYDRATOR-512</a> - Added to the Hydrator plugins a Sink plugin to write to
Solr search.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-517">HYDRATOR-517</a> - Added to the Hydrator plugins a Logistic Regression Spark
Machine Learning plugin.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-668">HYDRATOR-668</a> - Added to the Hydrator plugins a Decision Tree Regression
Spark Machine Learning plugin.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-909">HYDRATOR-909</a> - Added to the Hydrator plugins a SparkCompute Hydrator
plugin to compute N-Grams of any given String.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-935">HYDRATOR-935</a> - Added to the Hydrator plugins a Windows share copy Action
plugin.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-971">HYDRATOR-971</a> - Added to the Hydrator plugins a Hydrator plugin that
watches a directory and streams file content when new files are added.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-973">HYDRATOR-973</a> - Added to the Hydrator plugins an HTTP Poller source plugin
for streaming pipelines.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-977">HYDRATOR-977</a> - Added to the Hydrator plugins an XML parser plugin that can
parse out multiple records from a single XML document.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-981">HYDRATOR-981</a> - Added to the Hydrator plugins an Action plugin to run any
executable binary.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-1029">HYDRATOR-1029</a> - Added to the Hydrator plugins an Action plugin to export
data in an Oracle database.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-1091">HYDRATOR-1091</a> - Added the ability to run a Hydrator pipeline in a preview
mode without publishing. It allows users to view the data in each stage of the preview
run.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-1111">HYDRATOR-1111</a> - Added to the Hydrator plugins a plugin for transforming
data according to commands provided by the Cask Wrangler tool.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-1146">HYDRATOR-1146</a> - Added to the Hydrator plugins a Sink plugin to write to
Amazon Kinesis from Batch pipelines.</li>
</ul>
</li>
<li>Cask Tracker<ul>
<li><a class="reference external" href="https://issues.cask.co/browse/TRACKER-233">TRACKER-233</a> - Added a data dictionary to Cask Tracker for users to define
columns for datasets, enforce a common naming convention, and apply masking to PII
(personally identifiable information).</li>
</ul>
</li>
</ul>
</div>
<div class="section" id="id64">
<h3>Improvements<a class="headerlink" href="#id64" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-1280">CDAP-1280</a> - Merged various shell scripts into a single script to interface
with CDAP, called <code class="docutils literal notranslate"><span class="pre">cdap</span></code>, shipped with both the SDK and Distributed CDAP.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-1696">CDAP-1696</a> - Updated the default CDAP Router port to 11015 to avoid
conflicting with HiveServer2’s default port.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3262">CDAP-3262</a> - Fixed an issue with the CDAP scripts under Windows not
handling a JAVA_HOME path with spaces in it correctly. CDAP SDK home directories with
spaces in the path are not supported (due to issues with the product) and the scripts now
exit if such a path is detected.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4322">CDAP-4322</a> - For MapReduce programs using a PartitionedFileSet as input,
the partition key corresponding to the input split is now exposed to the mapper.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4901">CDAP-4901</a> - Fixed an issue where an exception from an HttpContentConsumer
was being silently ignored.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-5068">CDAP-5068</a> - Added pagination for the search RESTful API. Pagination is
achieved via <code class="docutils literal notranslate"><span class="pre">{{offset}}</span></code>, <code class="docutils literal notranslate"><span class="pre">{{limit}}`</span></code>, <code class="docutils literal notranslate"><span class="pre">{{numCursors}}</span></code>, and <code class="docutils literal notranslate"><span class="pre">{{cursor}}</span></code>
parameters in the RESTful API.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-5632">CDAP-5632</a> - New menu option in Cloudera Manager when running the CDAP CSD
enables running utilities such as the HBaseQueueDebugger.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-6183">CDAP-6183</a> - Added the property <code class="docutils literal notranslate"><span class="pre">program.container.dist.jars</span></code> to set
extra jars to be localized to every program container and to be added to classpaths of
CDAP programs.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-6425">CDAP-6425</a> - Fixed an issue that allowed a FileSet to be created if its
corresponding directory already existed.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-6572">CDAP-6572</a> - The namespace that integration test cases run against by
default has been made configurable.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-6577">CDAP-6577</a> - Improved the UpgradeTool to upgrade tables in namespaces with
impersonation configured.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-6587">CDAP-6587</a> - Added support for impersonation with CDAP Explore (Hive)
operations, including enabling exploring of a dataset or running queries against it.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-6635">CDAP-6635</a> - Added a feature that implements caching of user credentials in
CDAP system services.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-6837">CDAP-6837</a> - Fixed an issue in WorkerContext that did not properly
implement the contract of the Transactional interface. Note that this fix may cause
incompatibilities with previous releases in certain cases. See <a class="reference internal" href="#release-notes-cdap-6837"><span class="std std-ref">API Changes,
CDAP-6837</span></a> for more details.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-6862">CDAP-6862</a> - Updated more system services to respect the cdap-site
parameter “master.service.memory.mb”.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-6885">CDAP-6885</a> - Added support for concurrent runs of a Spark program.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-6937">CDAP-6937</a> - Added support for running CDAP on Apache HBase 1.2.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-6938">CDAP-6938</a> - Added support for Amazon EMR 4.6.0+ installation of CDAP via a
bootstrap action script.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-6984">CDAP-6984</a> - Added support for enabling SSL between the CDAP Router and
CDAP Master.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-6995">CDAP-6995</a> - Adding the capability to clean up log files which do not have
corresponding metadata.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7117">CDAP-7117</a> - Added support for checkpointing in Spark Streaming programs to
persist checkpoints transactionally.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7181">CDAP-7181</a> - Updated the Windows start scripts to match the new shell
script functionality.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7192">CDAP-7192</a> - Added the ability to specify an announce address and port for
the CDAP AppFabric and Dataset services. Deprecated the properties <code class="docutils literal notranslate"><span class="pre">app.bind.address</span></code>
and <code class="docutils literal notranslate"><span class="pre">dataset.service.bind.address</span></code>, replacing them with <code class="docutils literal notranslate"><span class="pre">master.services.bind.address</span></code>
as the bind address for master services. Added the properties
<code class="docutils literal notranslate"><span class="pre">master.services.announce.address</span></code>, <code class="docutils literal notranslate"><span class="pre">app.announce.port</span></code>, and
<code class="docutils literal notranslate"><span class="pre">dataset.service.announce.port</span></code> for use as announce addresses that are different from
the bind address.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7208">CDAP-7208</a> - Improved CDAP Master logging of events related to programs
that it launches.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7240">CDAP-7240</a> - Fixed a NullPointerException being logged on closing network
connection.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7284">CDAP-7284</a> - Upgraded the Apache Tephra version to 0.10-incubating.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7287">CDAP-7287</a> - Added support for enabling client certificate-based
authentication to the CDAP Authentication server.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7291">CDAP-7291</a> - Added support for CDH 5.9.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7319">CDAP-7319</a> - Provided programs more control over when and how transactions
are executed.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7385">CDAP-7385</a> - The Log HTTP Handler and Router have been fixed to allow the
streaming of larger logs files.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7393">CDAP-7393</a> - Revised the documentation on the recommended setting for
<code class="docutils literal notranslate"><span class="pre">yarn.nodemanager.delete.debug-delay-sec</span></code>.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7439">CDAP-7439</a> - Removed the requirement in the documentation of running
<code class="docutils literal notranslate"><span class="pre">kinit</span></code> prior to running the CDAP Upgrade Tool when upgrading a package installation of
CDAP on a secure Hadoop cluster.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7476">CDAP-7476</a> - Improves how MapReduce configures its inputs, such that
failures surface immediately.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7477">CDAP-7477</a> - Fixed an issue in MapReduce that caused skipping the
<code class="docutils literal notranslate"><span class="pre">destroy()</span></code> method if the committing of any of the dataset outputs failed.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7557">CDAP-7557</a> - <code class="docutils literal notranslate"><span class="pre">DynamicPartitioner</span></code> can now limit the number of open
RecordWriters to one, if the output partition keys are grouped.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7659">CDAP-7659</a> - Added support for specifying the Hive execution engine at
runtime (dynamically).</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7761">CDAP-7761</a> - Adds the <code class="docutils literal notranslate"><span class="pre">cluster.name</span></code> property that identifies a cluster;
this property can be set in the cdap-site.xml file.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7797">CDAP-7797</a> - Added a step in the CDAP Upgrade Tool to upgrade the
specification of the MetadataDataset.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-197">HYDRATOR-197</a> - Included an example of an action and post-run plugin in the
<code class="docutils literal notranslate"><span class="pre">cdap-data-pipeline-plugins-archetype</span></code>.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-947">HYDRATOR-947</a> - Improved the MockSource unit test plugin so that it can be
configured to set an output schema, allowing subsequent plugins in the pipeline to have
non-null input schemas.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-966">HYDRATOR-966</a> - Enabled macros for the Hive database, table name, and
metastore URI properties for the Hive plugins.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-976">HYDRATOR-976</a> - Added compression options to the HDFS sink plugin.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-996">HYDRATOR-996</a> - Enhanced the Kafka streaming source to support configurable
partitions and initial offsets, and to support optionally including the partition and
offset in the output records.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-1004">HYDRATOR-1004</a> - The File Batch source in Hydrator now ignores empty
directories.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-1069">HYDRATOR-1069</a> - The CSV parser can now accept a custom delimiter for
parsing CSV files.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-1072">HYDRATOR-1072</a> - The Script filter plugin has been removed from Hydrator;
the JavaScript filter can be used instead.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/TRACKER-167">TRACKER-167</a> - Cask Tracker now includes “unknown” accesses when finding
top datasets.</li>
</ul>
</div>
<div class="section" id="id65">
<h3>Bug Fixes<a class="headerlink" href="#id65" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2945">CDAP-2945</a> - A MapReduce job using either a FileSet or PartitionedFileSet
as input no longer fails if there are no input partitions.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4535">CDAP-4535</a> - The Authentication server announce address is now
configurable.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-5012">CDAP-5012</a> - Fixed a problem with downloading of large (multiple gigabyte)
CDAP Explore queries.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-5061">CDAP-5061</a> - Fixed an issue where the metadata of streams was not being
updated when the stream’s schema was altered.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-5372">CDAP-5372</a> - Fixed an issue where a warning was logged instead of an error
when a MapReduce job failed in the CDAP SDK.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-5897">CDAP-5897</a> - Updated the default CDAP UI port to 11011 to avoid conflicting
with Accumulo and Cloudera Manager’s Activity Monitor.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-6398">CDAP-6398</a> - Authentication handler APIs have been updated to restrict
which <code class="docutils literal notranslate"><span class="pre">cdap-site.xml</span></code> and <code class="docutils literal notranslate"><span class="pre">cdap-security.xml</span></code> properties are available to it.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-6404">CDAP-6404</a> - Fixed an issue with searching for an entity in Cask Tracker by
metadata after a tag with the same prefix has been removed.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7031">CDAP-7031</a> - Fixed an issue with misleading log messages from the RunRecord
corrector.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7116">CDAP-7116</a> - Fixed an issue so as to significantly reduce the chance of a
schedule misfire in the case where the CPU cannot trigger a schedule within a certain time
threshold.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7138">CDAP-7138</a> - Fixed a problem with duplicate logs showing for a running
program.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7154">CDAP-7154</a> - On an incorrect ZooKeeper quorum configuration, the CDAP
Upgrade Tool and other services such as Master, Router, and Kafka will timeout with an
error instead of hanging indefinitely.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7175">CDAP-7175</a> - Fixed an issue in the CDAP Upgrade Tool to allow it to run on
a CDAP instance with authorization enabled.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7177">CDAP-7177</a> - Fixed an issue where macros were not being substituted for
postaction plugins.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7204">CDAP-7204</a> - Lineage information is now returned for deleted datasets.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7248">CDAP-7248</a> - Fixed an issue with the FileBatchSource not working with Azure
Blob Storage.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7249">CDAP-7249</a> - Fixed an issue with CDAP Explore using Tez on Azure HDInsight.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7250">CDAP-7250</a> - Fixed an issue where dataset usage was not being recorded
after an application was deleted.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7256">CDAP-7256</a> - Fixed an issue with the leaking of Hive classes to programs in
the CDAP SDK.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7259">CDAP-7259</a> - Added a warning when a PartitionFilter addresses a
non-existent field.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7285">CDAP-7285</a> - Fixed an issue that prevented launching of MapReduce jobs on a
Hadoop-2.7 cluster.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7292">CDAP-7292</a> - Fixed an issue in the KMeans example that caused it to
calculate the wrong cluster centroids.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7314">CDAP-7314</a> - Fixed an issue with the documentation example links to the
CDAP ETL Guide.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7317">CDAP-7317</a> - Fixed a misleading error message that occurred when the
updating of a CDAP Explore table for a dataset failed.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7318">CDAP-7318</a> - Fixed an issue that would cause MapReduce and Spark programs
to fail if too many macros were being used.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7321">CDAP-7321</a> - Fixed an issue with upgrading CDAP using the CDAP Upgrade
Tool.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7324">CDAP-7324</a> - Fixed an issue with the CDAP Upgrade Tool while upgrading
HBase coprocessors.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7361">CDAP-7361</a> - Fixed an issue with log file corruption if the log saver
container crashed due to being killed by YARN.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7374">CDAP-7374</a> - Fixed an issue with Hydrator Studio in the Windows version of
Chrome that prevented users from opening and editing a node configuration.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7394">CDAP-7394</a> - Fixed an issue that prevented impersonation in flows from
working correctly, by not re-using HBaseAdmin across different UGI.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7417">CDAP-7417</a> - Fixes an issue where the partitions of a PartitionedFileSet
were not cleaned up properly after a transaction failure.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7428">CDAP-7428</a> - Fixed an issue preventing having CustomAction and Spark as
inner classes.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7442">CDAP-7442</a> - CDAP Ambari Service’s required version of Ambari Server was
increased to 2.2 to support the empty-value-valid configuration attribute.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7473">CDAP-7473</a> - Fix the logback-container.xml to work on clusters with
multiple log directories configured for YARN.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7482">CDAP-7482</a> - Fixed an issue in CDAP logging that caused system logs from
Kafka to not be saved after an upgrade and for previously-saved logs to become
inaccessible.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7483">CDAP-7483</a> - Fixes an issue where a MapReduce using DynamicPartitioner
would leave behind output files if it failed.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7500">CDAP-7500</a> - Fixed an issue where a MapReduce classloader gets closed
prematurely.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7514">CDAP-7514</a> - Fixed an issue preventing proper class loading isolation for
explicit transactions executed by programs.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7522">CDAP-7522</a> - Improved the documentation for read-less increments.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7524">CDAP-7524</a> - Adds a missing <code class="docutils literal notranslate"><span class="pre">&#64;Override</span></code> annotation for the
<code class="docutils literal notranslate"><span class="pre">WorkerContext.execute()</span></code> method.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7527">CDAP-7527</a> - Fixed an issue that prevented the using of the logback.xml
from an application JAR.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7548">CDAP-7548</a> - Fixed an issue in integration tests to allow JDBC connections
against authorization-enabled and SSL-enabled CDAP instances.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7566">CDAP-7566</a> - Improved the usability of ServiceManager in integration tests.
The <code class="docutils literal notranslate"><span class="pre">getServiceURL()</span></code> method now waits for the service to be discoverable before
returning the service’s URL.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7612">CDAP-7612</a> - Fixed an issue where Spark programs could not be started after
a master failover or restart.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7624">CDAP-7624</a> - Fixed an issue where readless increments from different
MapReduce tasks cancelled each other out.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7629">CDAP-7629</a> - Added additional tests for read-less increments in HBase.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7648">CDAP-7648</a>, <a class="reference external" href="https://issues.cask.co/browse/CDAP-7663">CDAP-7663</a> - Added support for Amazon EMR 4.6.0.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7652">CDAP-7652</a> - Startup checks now validate the HBase version and error out if
the HBase version is not supported.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7660">CDAP-7660</a> - The CDAP Ambari service was updated to use scripts for Auth
Server/Router alerts in Ambari due to Ambari not supporting CDAP’s <code class="docutils literal notranslate"><span class="pre">/status</span></code> endpoint
with WEB check.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7664">CDAP-7664</a> - CDAP Quick Links in the CDAP Ambari Service now correctly link
to the CDAP UI.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7666">CDAP-7666</a> - Fixed the YARN startup check to fail instead of warning if the
cluster does not have enough capacity to run CDAP services.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7680">CDAP-7680</a> - Fixed an issue in the CDAP Sentry Extension by which
privileges were not being deleted when the CDAP entity was deleted.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7707">CDAP-7707</a> - Files installed by the “cdap” package under <code class="docutils literal notranslate"><span class="pre">/etc</span></code> are now
properly marked as <code class="docutils literal notranslate"><span class="pre">config</span></code> files for RPM packages.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7724">CDAP-7724</a> - Fixed an issue that could cause Spark and MapReduce programs
to stop improperly, resulting in a failed run record instead of a killed run record.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7737">CDAP-7737</a> - Fixed the <code class="docutils literal notranslate"><span class="pre">cdap-data-pipeline-plugins-archetype</span></code> to export
everything in the provided <code class="docutils literal notranslate"><span class="pre">groupId</span></code> and fixed the archetype to use the provided
<code class="docutils literal notranslate"><span class="pre">groupId</span></code> as the Java package instead of using a hardcoded value.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7742">CDAP-7742</a> - Fixed the ordering of search results by relevance in the
search RESTful API.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7757">CDAP-7757</a> - Now uses the OpenJDK for redistributable images, such as
Docker and Virtual Machine images.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7819">CDAP-7819</a> - The Node.js version check in the CDAP SDK was updated to
properly handle patch-level comparisons.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-89">HYDRATOR-89</a> - Batch Hydrator pipelines will now log an error instead of a
warning if they fail in the CDAP SDK.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-471">HYDRATOR-471</a> - The Database Batch Source now handles $CONDITIONS when
getting a schema.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-499">HYDRATOR-499</a> - GetSchema for an aggregator now fails if there are
duplicate names.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-791">HYDRATOR-791</a> - Fixed an issue where Hydrator pipelines using a DBSource
were not working in an HDP cluster.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-915">HYDRATOR-915</a> - Fixed an issue where pipelines with multiple sinks
connected to the same action could fail to publish.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-948">HYDRATOR-948</a> - Fixed an issue with Spark data pipelines not supporting
argument values in excess of 64K characters.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-950">HYDRATOR-950</a> - Password field is now masked in the Email post-run plugin.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-968">HYDRATOR-968</a> - Fixed an issue so that the CDAP UI does not parse macros
when starting a pipeline in Hydrator.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-978">HYDRATOR-978</a> - Fixed an issue where macros were not being evaluated in
streaming source Hydrator plugins.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-987">HYDRATOR-987</a> - Fixed the UI widget for the S3 source to make its output
schema non-editable.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-994">HYDRATOR-994</a> - Stream source duration in the stream source hydrator plugin
is now macro-enabled.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-1010">HYDRATOR-1010</a> - The Python evaluator can now handle float and double data
types.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-1025">HYDRATOR-1025</a> - Fixed an issue to format XML correctly in the XML reader
plugin.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-1062">HYDRATOR-1062</a> - Fixed a serialization issue with StructuredRecords that
use primitive arrays.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-1126">HYDRATOR-1126</a> - Fixed an issue where the outputSchema plugin function
expected an input schema to be present.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-1131">HYDRATOR-1131</a> - Added being able to add to an error dataset for malformed
rows in CSV while parsing using the CSV parser.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-1132">HYDRATOR-1132</a> - A Hydrator application can now set <em>reducer</em> task
resources as a per-worker resource provided for MapReduce pipelines.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-1168">HYDRATOR-1168</a> - Spark pipelines now use 1024mb of memory by default for
the Spark client that submits the job.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-1189">HYDRATOR-1189</a> - Any Hydrator pipelines that use S3 (either as an S3 source
or an S3 sink) based on core-plugins version 1.4 (used in CDAP prior to 4.0.0) will not
execute on a 4.0.x cluster. A workaround is to recreate (clone) the pipeline using a newer
version of core-plugins (version 1.5 or higher).</li>
<li><a class="reference external" href="https://issues.cask.co/browse/TRACKER-217">TRACKER-217</a> - Fixed an issue preventing the adding of additional tags
after an existing tag had been deleted.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/TRACKER-225">TRACKER-225</a> - Fixed an issue where Cask Tracker was creating too many
connections to ZooKeeper.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/TRACKER-229">TRACKER-229</a> - Fixed an issue that was sending program run ids instead of
program names.</li>
</ul>
</div>
<div class="section" id="id66">
<h3>Known Issues<a class="headerlink" href="#id66" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-6099">CDAP-6099</a> - Due to a limitation in the CDAP MapReduce implementation,
writing to a dataset does not work in a MapReduce Mapper’s <code class="docutils literal notranslate"><span class="pre">destroy()</span></code> method.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7444">CDAP-7444</a> - If a MapReduce program fails during startup, the program’s
<code class="docutils literal notranslate"><span class="pre">destroy()</span></code> method is never called, preventing any cleanup or action there being taken.</li>
</ul>
</div>
<div class="section" id="id67">
<h3>API Changes<a class="headerlink" href="#id67" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-1696">CDAP-1696</a> - <strong>Updated the default CDAP Router port to 11015</strong> to avoid
conflicting with HiveServer2’s default port. <strong>Note that this change may cause
incompatibilities with previous releases if hardcoded in scripts or other programs.</strong></li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-5897">CDAP-5897</a> - <strong>Updated the default CDAP UI port to 11011</strong> to avoid
conflicting with Accumulo and Cloudera Manager’s Activity Monitor. <strong>Note that this change
may cause incompatibilities with previous releases if hardcoded in scripts or other
programs.</strong></li>
</ul>
<ul id="release-notes-cdap-6837">
<li><p class="first"><a class="reference external" href="https://issues.cask.co/browse/CDAP-6837">CDAP-6837</a> - Fixed an issue in <code class="docutils literal notranslate"><span class="pre">WorkerContext</span></code> that did not properly
implement the contract of the Transactional interface. <strong>Note that this fix may cause
incompatibilities with previous releases in certain cases.</strong> See below for details on
how to handle this change in existing code.</p>
<p>The Transactional API defines:</p>
<div class="highlight-java notranslate"><div class="highlight"><pre><span></span><span class="kt">void</span> <span class="nf">execute</span><span class="p">(</span><span class="n">TxRunnable</span> <span class="n">runnable</span><span class="p">)</span> <span class="kd">throws</span> <span class="n">TransactionFailureException</span><span class="p">;</span>
</pre></div>
</div>
<p>and <code class="docutils literal notranslate"><span class="pre">WorkerContext</span></code> implements <code class="docutils literal notranslate"><span class="pre">Transactional</span></code>. However, it declares this method to
not throw checked exceptions:</p>
<div class="highlight-java notranslate"><div class="highlight"><pre><span></span><span class="kt">void</span> <span class="nf">execute</span><span class="p">(</span><span class="n">TxRunnable</span> <span class="n">runnable</span><span class="p">);</span>
</pre></div>
</div>
<p>That means that any <code class="docutils literal notranslate"><span class="pre">TransactionFailureException</span></code> thrown from a
<code class="docutils literal notranslate"><span class="pre">WorkerContext.execute()</span></code> is wrapped into a <code class="docutils literal notranslate"><span class="pre">RuntimeException</span></code>, and callers must
write code similar to this to handle the exception:</p>
<div class="highlight-java notranslate"><div class="highlight"><pre><span></span><span class="k">try</span> <span class="p">{</span>
  <span class="n">getContext</span><span class="p">().</span><span class="na">execute</span><span class="p">(...);</span>
<span class="p">}</span> <span class="k">catch</span> <span class="p">(</span><span class="n">Exception</span> <span class="n">e</span><span class="p">)</span> <span class="p">{</span>
  <span class="k">if</span> <span class="p">(</span><span class="n">e</span><span class="p">.</span><span class="na">getCause</span><span class="p">()</span> <span class="k">instanceof</span> <span class="n">TransactionFailureException</span><span class="p">)</span> <span class="p">{</span>
    <span class="c1">// Handle it</span>
  <span class="p">}</span> <span class="k">else</span> <span class="p">{</span>
    <span class="c1">// What else to expect? It&#39;s not clear...</span>
    <span class="k">throw</span> <span class="n">Throwables</span><span class="p">.</span><span class="na">propagate</span><span class="p">(</span><span class="n">e</span><span class="p">);</span>
  <span class="p">}</span>
<span class="p">}</span>
</pre></div>
</div>
<p>This is ugly and inconsistent with other implementations of Transactional. We have
addressed this by altering the <code class="docutils literal notranslate"><span class="pre">WorkerContext</span></code> to directly raise the
<code class="docutils literal notranslate"><span class="pre">TransactionFailureException</span></code>. <strong>However, code must change to accomodate this.</strong></p>
<p>To address this in existing code, such that it will work both in 4.0.0 and earlier
versions of CDAP, use code similar to this:</p>
<div class="highlight-java notranslate"><div class="highlight"><pre><span></span><span class="nd">@Override</span>
<span class="kd">public</span> <span class="kt">void</span> <span class="nf">run</span><span class="p">()</span> <span class="p">{</span>
  <span class="k">try</span> <span class="p">{</span>
    <span class="n">getContext</span><span class="p">().</span><span class="na">execute</span><span class="p">(</span><span class="k">new</span> <span class="n">TxRunnable</span><span class="p">()</span> <span class="p">{</span>
      <span class="nd">@Override</span>
      <span class="kd">public</span> <span class="kt">void</span> <span class="nf">run</span><span class="p">(</span><span class="n">DatasetContext</span> <span class="n">context</span><span class="p">)</span> <span class="kd">throws</span> <span class="n">Exception</span> <span class="p">{</span>
        <span class="k">if</span> <span class="p">(</span><span class="n">getContext</span><span class="p">().</span><span class="na">getRuntimeArguments</span><span class="p">().</span><span class="na">containsKey</span><span class="p">(</span><span class="s">&quot;fail&quot;</span><span class="p">))</span> <span class="p">{</span>
          <span class="k">throw</span> <span class="k">new</span> <span class="n">RuntimeException</span><span class="p">(</span><span class="s">&quot;fail&quot;</span><span class="p">);</span>
        <span class="p">}</span>
      <span class="p">}</span>
    <span class="p">});</span>
  <span class="p">}</span> <span class="k">catch</span> <span class="p">(</span><span class="n">Exception</span> <span class="n">e</span><span class="p">)</span> <span class="p">{</span>
    <span class="k">if</span> <span class="p">(</span><span class="n">e</span> <span class="k">instanceof</span> <span class="n">TransactionFailureException</span><span class="p">)</span> <span class="p">{</span>
      <span class="n">LOG</span><span class="p">.</span><span class="na">error</span><span class="p">(</span><span class="s">&quot;transaction failure&quot;</span><span class="p">);</span>
    <span class="p">}</span> <span class="k">else</span> <span class="k">if</span> <span class="p">(</span><span class="n">e</span><span class="p">.</span><span class="na">getCause</span><span class="p">()</span> <span class="k">instanceof</span> <span class="n">TransactionFailureException</span><span class="p">)</span> <span class="p">{</span>
      <span class="n">LOG</span><span class="p">.</span><span class="na">error</span><span class="p">(</span><span class="s">&quot;exception with cause transaction failure&quot;</span><span class="p">);</span>
    <span class="p">}</span> <span class="k">else</span> <span class="p">{</span>
      <span class="n">LOG</span><span class="p">.</span><span class="na">error</span><span class="p">(</span><span class="s">&quot;other failure&quot;</span><span class="p">);</span>
    <span class="p">}</span>
  <span class="p">}</span>
<span class="p">}</span>
</pre></div>
</div>
<p>This code will succeed because it handles both the “new style” of the <code class="docutils literal notranslate"><span class="pre">WorkerContext</span></code>
directly throwing a <code class="docutils literal notranslate"><span class="pre">TransactionFailureException</span></code> and at the same time handle the
previous style of the <code class="docutils literal notranslate"><span class="pre">TransactionFailureException</span></code> being wrapped in a
<code class="docutils literal notranslate"><span class="pre">RuntimeException</span></code>.</p>
<p>Code that is only used in CDAP 4.0.0 and higher can use a simpler version of this:</p>
<div class="highlight-java notranslate"><div class="highlight"><pre><span></span>  <span class="nd">@Override</span>
  <span class="kd">public</span> <span class="kt">void</span> <span class="nf">run</span><span class="p">()</span> <span class="p">{</span>
    <span class="k">try</span> <span class="p">{</span>
      <span class="n">getContext</span><span class="p">().</span><span class="na">execute</span><span class="p">(</span><span class="k">new</span> <span class="n">TxRunnable</span><span class="p">()</span> <span class="p">{</span>
        <span class="nd">@Override</span>
        <span class="kd">public</span> <span class="kt">void</span> <span class="nf">run</span><span class="p">(</span><span class="n">DatasetContext</span> <span class="n">context</span><span class="p">)</span> <span class="kd">throws</span> <span class="n">Exception</span> <span class="p">{</span>
          <span class="k">if</span> <span class="p">(</span><span class="n">getContext</span><span class="p">().</span><span class="na">getRuntimeArguments</span><span class="p">().</span><span class="na">containsKey</span><span class="p">(</span><span class="s">&quot;fail&quot;</span><span class="p">))</span> <span class="p">{</span>
            <span class="k">throw</span> <span class="k">new</span> <span class="n">RuntimeException</span><span class="p">(</span><span class="s">&quot;fail&quot;</span><span class="p">);</span>
          <span class="p">}</span>
        <span class="p">}</span>
      <span class="p">});</span>
    <span class="p">}</span> <span class="k">catch</span> <span class="p">(</span><span class="n">TransactionFailureException</span> <span class="n">e</span><span class="p">)</span> <span class="p">{</span>
      <span class="p">...</span>
    <span class="p">}</span>
  <span class="p">}</span>
<span class="p">}</span>
</pre></div>
</div>
</li>
<li><p class="first"><a class="reference external" href="https://issues.cask.co/browse/CDAP-7544">CDAP-7544</a> - The <a class="reference external" href="http://docs.cask.co/cdap/4.0.0/en/reference-manual/http-restful-api/metadata.html#http-restful-api-metadata-searching">Metadata HTTP RESTful API</a>
has been modified to support sorting and
pagination. To do so, the API now uses additional parameters—<code class="docutils literal notranslate"><span class="pre">sort</span></code>, <code class="docutils literal notranslate"><span class="pre">offset</span></code>,
<code class="docutils literal notranslate"><span class="pre">limit</span></code>, <code class="docutils literal notranslate"><span class="pre">numCursors</span></code>, and <code class="docutils literal notranslate"><span class="pre">cursor</span></code>—and the format of the results
returned when searching has changed. Whereas previous to CDAP 4.0.0 the API returned
results as a list of results, the API now returns the results as a field in a JSON object.</p>
</li>
<li><p class="first"><a class="reference external" href="https://issues.cask.co/browse/CDAP-7796">CDAP-7796</a> - Two properties are changing in version 4.0.0 of the CSD:</p>
<ul class="simple">
<li><code class="docutils literal notranslate"><span class="pre">log.saver.run.memory.megs</span></code> is replaced with <code class="docutils literal notranslate"><span class="pre">log.saver.container.memory.mb</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">log.saver.run.num.cores</span></code> is replaced with <code class="docutils literal notranslate"><span class="pre">log.saver.container.num.cores</span></code></li>
</ul>
<p>Anyone who has modified these properties in previous versions will have to update them
after upgrading.</p>
</li>
</ul>
</div>
<div class="section" id="id68">
<h3>Deprecated and Removed Features<a class="headerlink" href="#id68" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-5246">CDAP-5246</a> - Removed the deprecated Kafka feed for metadata updates. Users
should instead subscribe to the CDAP Audit feed, which contains metadata update
notifications in messages with audit type <code class="docutils literal notranslate"><span class="pre">METADATA_CHANGE</span></code>.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-6862">CDAP-6862</a> - Deprecated “log.saver.run.memory.megs” and
“log.saver.run.num.cores”, in favor of “log.saver.container.memory.mb” and
“log.saver.container.num.cores”, respectively.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7475">CDAP-7475</a> - Removes deprecated methods <code class="docutils literal notranslate"><span class="pre">setInputDataset()</span></code>,
<code class="docutils literal notranslate"><span class="pre">setOutputDataset()</span></code>, and <code class="docutils literal notranslate"><span class="pre">useStreamInput()</span></code> from the MapReduce API, and related
methods from the MapReduceContext.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7718">CDAP-7718</a> - Removed the deprecated <code class="docutils literal notranslate"><span class="pre">StreamBatchReadable</span></code> class.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7127">CDAP-7127</a> - The deprecated CDAP Explore service instance property has been
removed.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7205">CDAP-7205</a> - Removes the deprecated <code class="docutils literal notranslate"><span class="pre">useDatasets()</span></code> method from API and
documentation.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7563">CDAP-7563</a> - Removed the usage of deprecated methods from examples.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-1094">HYDRATOR-1094</a> - Removed the deprecated
<code class="docutils literal notranslate"><span class="pre">cdap-etl-batch-source-archetype</span></code>, <code class="docutils literal notranslate"><span class="pre">cdap-etl-batch-sink-archetype</span></code>, and
<code class="docutils literal notranslate"><span class="pre">cdap-etl-transform-archetype</span></code> in favor of the <code class="docutils literal notranslate"><span class="pre">cdap-data-pipeline-plugins-archetype</span></code>.</li>
</ul>
</div>
</div>
<div class="section" id="release-3-6-0">
<h2><a class="reference external" href="http://docs.cask.co/cdap/3.6.0/index.html">Release 3.6.0</a><a class="headerlink" href="#release-3-6-0" title="Permalink to this headline">🔗</a></h2>
<div class="section" id="id69">
<h3>Improvements<a class="headerlink" href="#id69" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-5771">CDAP-5771</a> - Allow concurrent runs of different versions of a service. A
RouteConfig can be uploaded to configure the percentage of requests that need to be sent
to the different versions.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7281">CDAP-7281</a> - Improved the PartitionedFileSet to validate the schema of a
partition key. Note that this will break code that uses incorrect partition keys, which
was previously silently ignored.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7343">CDAP-7343</a> - All non-versioned endpoints are now directed to applications
with a default version. Added test cases with a mixed usage of the new versioned endpoints
and the corresponding non-versioned endpoints.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7366">CDAP-7366</a> - Added an upgrade step that adds a default version ID to jobs
and triggers in the Schedule Store.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7385">CDAP-7385</a> - The Log HTTP Handler and Router have been fixed to allow the
streaming of larger logs files.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7264">CDAP-7264</a> - Added an HTTP RESTful API to create applications with a
version.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7265">CDAP-7265</a> - Added an HTTP RESTful API to start or stop programs of a
specific application version.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7266">CDAP-7266</a> - Added an upgrade step that adds a default application version
to existing applications.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7268">CDAP-7268</a> - Added an HTTP RESTful API to store, fetch, and delete
RouteConfigs for user service endpoint routing control.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7272">CDAP-7272</a> - User services now include their application version in the
payload when they announce themselves in Apache Twill.</li>
</ul>
</div>
<div class="section" id="id70">
<h3>Bug Fixes<a class="headerlink" href="#id70" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3822">CDAP-3822</a> - Unit Test framework now has the capability to exclude scala,
so users can depend on their own version of the library.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7250">CDAP-7250</a> - Fixed an issue where dataset usage was not being recorded
after an application was deleted.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7314">CDAP-7314</a> - Fixed a problem with the documentation example links to the
CDAP ETL Guide.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7321">CDAP-7321</a> - Fixed a problem with upgrading CDAP using the CDAP Upgrade
Tool.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7324">CDAP-7324</a> - Fixed a problem with the upgrade tool while upgrading HBase
coprocessors.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7334">CDAP-7334</a> - Fixed a problem with the listing of applications not returning
the application version correctly.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7353">CDAP-7353</a> - Fixed a problem with using “Download All” logs in the
browser log viewer by having it fetch and stream the response to the client.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7359">CDAP-7359</a> - Fixed a problem with NodeJS buffering a response before
sending it to a client.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7361">CDAP-7361</a> - Fixed a problem with log file corruption if the log saver
container crashes due to being killed by YARN.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7364">CDAP-7364</a> - Fixed a problem with the CDAP UI not handling “5xx” error
codes correctly.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7374">CDAP-7374</a> - Fixed Hydrator Studio in the Windows version of Chrome to
allow users to open and edit a node configuration.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7386">CDAP-7386</a> - Fixed an error in the “CDAP Introduction” tutorial’s
“Transforming Your Data” example of an application configuration.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7391">CDAP-7391</a> - Fixed an issue that caused unit test failures when using
<code class="docutils literal notranslate"><span class="pre">org.hamcrest</span></code> classes.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7392">CDAP-7392</a> - Fixed an issue where the Java process corresponding to the
MapReduce application master kept running even if the application was moved to the FINISHED
state.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-791">HYDRATOR-791</a> - Fixed a problem with Hydrator pipelines using a DBSource
not working in an HDP cluster.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-948">HYDRATOR-948</a> - Fixed a problem with Spark data pipelines not supporting
argument values in excess of 64K characters.</li>
</ul>
</div>
</div>
<div class="section" id="release-3-5-2">
<h2><a class="reference external" href="http://docs.cask.co/cdap/3.5.2/index.html">Release 3.5.2</a><a class="headerlink" href="#release-3-5-2" title="Permalink to this headline">🔗</a></h2>
<div class="section" id="id71">
<h3>Known Issues<a class="headerlink" href="#id71" title="Permalink to this headline">🔗</a></h3>
<ul>
<li><p class="first"><a class="reference external" href="https://issues.cask.co/browse/CDAP-7179">CDAP-7179</a> - In CDAP 3.5.0, new <code class="docutils literal notranslate"><span class="pre">kafka.server.*</span></code> properties replace older
properties such as <code class="docutils literal notranslate"><span class="pre">kafka.log.dir</span></code>, as described in the <a class="reference external" href="../../admin-manual/appendices/cdap-site.html#appendix-cdap-default-deprecated-properties" title="(in Cask Data Application Platform v6.1.1)"><span class="xref std std-ref">管理手册:
Appendices: cdap-site.xml</span></a>.</p>
<p><strong>If you are upgrading from CDAP 3.4.x to 3.5.x</strong> and you have set a value for
<code class="docutils literal notranslate"><span class="pre">kafka.log.dir</span></code> by using Cloudera Manager’s <a class="reference external" href="../../admin-manual/installation/cloudera.html#cloudera-installation-add-service-wizard-configuration" title="(in Cask Data Application Platform v6.1.1)"><span class="xref std std-ref">safety-valve mechanism</span></a>, you need to change to the new
property <code class="docutils literal notranslate"><span class="pre">kafka.server.log.dirs</span></code>, as the deprecated <code class="docutils literal notranslate"><span class="pre">kafka.log.dir</span></code> is being ignored
in favor of the new property. If you don’t, your custom value will be replaced with the
default value.</p>
</li>
<li><p class="first"><a class="reference external" href="https://issues.cask.co/browse/CDAP-7608">CDAP-7608</a> - When running in Standalone CDAP, the Cask Hydrator plugin
NaiveBayesTrainer has a <em>permgen</em> memory leak that leads to an out-of-memory error if
the plugin is repeatedly used a number of times, as few as six runs. The only workaround
is to reset the memory by restarting Standalone CDAP.</p>
</li>
</ul>
</div>
<div class="section" id="id72">
<h3>Improvements<a class="headerlink" href="#id72" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3262">CDAP-3262</a> - Fixed an issue with the CDAP scripts under Windows not
handling a JAVA_HOME path with spaces in it correctly. CDAP SDK home directories with
spaces in the path are not supported (due to issues with the product) and the scripts now
exit if such a path is detected.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4322">CDAP-4322</a> - For MapReduce programs using a PartitionedFileSet as input,
expose the partition key corresponding to the input split to the mapper.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-6183">CDAP-6183</a> - Added the property <code class="docutils literal notranslate"><span class="pre">program.container.dist.jars</span></code> to set
extra jars to be localized to every program container and to be added to classpaths of
CDAP programs.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-6572">CDAP-6572</a> - The namespace that integration test cases run against by
default has been made configurable.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-6577">CDAP-6577</a> - Improve UpgradeTool to upgrade tables in namespaces with
impersonation configured.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-6885">CDAP-6885</a> - Added support for concurrent runs of a Spark program.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-6587">CDAP-6587</a> - Added support for impersonation with CDAP Explore (Hive)
operations, such as enabling exploring of a dataset or running queries against it.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7291">CDAP-7291</a> - Added support for CDH 5.9.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7385">CDAP-7385</a> - The Log HTTP Handler and Router have been fixed to allow the
streaming of larger logs files.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7387">CDAP-7387</a> - Added support to LogSaver for impersonation.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7404">CDAP-7404</a> - Added authorization for schedules in CDAP.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7529">CDAP-7529</a> - Improved error handling upon failures in namespace creation.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7557">CDAP-7557</a> - DynamicPartitioner can now limit the number of open
RecordWriters to one, if the output partition keys are grouped.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7682">CDAP-7682</a> - Added a property <code class="docutils literal notranslate"><span class="pre">kafka.zookeeper.quorum</span></code> to be used across
all internal clients using Kafka.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7761">CDAP-7761</a> - Adds <code class="docutils literal notranslate"><span class="pre">cluster.name</span></code> as a property that identifies a cluster;
this property can be set in the <code class="docutils literal notranslate"><span class="pre">cdap-site.xml</span></code>.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-979">HYDRATOR-979</a> - Added the Windows Share Copy plugin to the Hydrator plugins.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-997">HYDRATOR-997</a> - The SSH hostname and the command to be executed are now
macro-enabled for the SSH action plugin.</li>
</ul>
</div>
<div class="section" id="id73">
<h3>Bug Fixes<a class="headerlink" href="#id73" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-6981">CDAP-6981</a> - Fixed an issue that prevented macros from being used with a
secure KMS store.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7116">CDAP-7116</a> - Fixed an issue so as to significantly reduce the chance of a
schedule misfire in the case where the CPU cannot trigger a schedule within a certain time
threshold.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7177">CDAP-7177</a> - Fixed an issue where macros were not being substituted for
postaction plugins.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7250">CDAP-7250</a> - Fixed an issue where dataset usage was not being recorded
after an application was deleted.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7318">CDAP-7318</a> - Fixed an issue that would cause MapReduce and Spark programs
to fail if too many macros were being used.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7391">CDAP-7391</a> - Fixed TestFramework classloading to support classes that
depend on <code class="docutils literal notranslate"><span class="pre">org.hamcrest</span></code>.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7392">CDAP-7392</a> - Fixed an issue where the Java process corresponding to the
MapReduce application master kept running even if the application was moved to the
FINISHED state.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7394">CDAP-7394</a> - Fixed an issue with impersonation in flows not working by not
re-using <code class="docutils literal notranslate"><span class="pre">HBaseAdmin</span></code> across different UGI.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7396">CDAP-7396</a> - Fixed an issue which prevented scheduled jobs from running on
a namespace with impersonation.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7398">CDAP-7398</a> - Fixed an issue which prevented an app in a namespace from
being deleted if a program for the same app is running in a different namespace.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7403">CDAP-7403</a> - Fixed an issue that prevented the CDAP UI from starting if the
<code class="docutils literal notranslate"><span class="pre">logback.xml</span></code> was configured to log at the INFO or lower level.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7404">CDAP-7404</a> - Added authorization for schedules in CDAP.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7420">CDAP-7420</a> - Avoid the caching of <code class="docutils literal notranslate"><span class="pre">YarnClient</span></code> in order to fix a problem
that occurred in namespaces with impersonation configured.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7433">CDAP-7433</a> - Fixed an issue that prevented <code class="docutils literal notranslate"><span class="pre">HBaseQueueDebugger</span></code> from
running in an impersonated namespace.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7435">CDAP-7435</a> - Fixed an error which prevented the downloading of large logs
using the CDAP UI.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7438">CDAP-7438</a>, <a class="reference external" href="https://issues.cask.co/browse/CDAP-7439">CDAP-7439</a> - Removed the requirement of running
“kinit” prior to running either the Upgrade or Transaction Debugger tools of CDAP on a
secure Hadoop cluster.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7458">CDAP-7458</a> - Fixed an issue that prevented the CDAP Upgrade Tool from being
run for a namespace with authorization turned on.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7473">CDAP-7473</a> - Fix logback-container.xml to work on clusters with multiple
log directories configured for YARN.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7482">CDAP-7482</a> - Fixed a problem in CDAP logging that caused system logs from
Kafka to not be saved after an upgrade and for previously-saved logs to become
inaccessible.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7500">CDAP-7500</a> - Fixed cases where the MapReduce classloader was being closed
prematurely.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7527">CDAP-7527</a> - Fixed a problem that prevented the use of a <code class="docutils literal notranslate"><span class="pre">logback.xml</span></code>
from an application jar.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7548">CDAP-7548</a> - Fixed a problem in integration tests to allow JDBC connections
against authorization-enabled and SSL-enabled CDAP instances.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7566">CDAP-7566</a> - Improved the usability of ServiceManager in integration tests.
The <code class="docutils literal notranslate"><span class="pre">getServiceURL</span></code> method now waits for the service to be discoverable before
returning the service’s URL.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7612">CDAP-7612</a> - Fixed cases where Spark programs could not be started after a
master failover or restart.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7660">CDAP-7660</a> - The CDAP Ambari service was updated to use scripts for Auth
Server/Router alerts in Ambari due to Ambari not supporting CDAP’s <code class="docutils literal notranslate"><span class="pre">/status</span></code> endpoint
with WEB check.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-1125">HYDRATOR-1125</a> - Fixed a problem that prevented the adding of a schema with
hyphens in the Hydrator UI.</li>
</ul>
</div>
</div>
<div class="section" id="release-3-5-1">
<h2><a class="reference external" href="http://docs.cask.co/cdap/3.5.1/index.html">Release 3.5.1</a><a class="headerlink" href="#release-3-5-1" title="Permalink to this headline">🔗</a></h2>
<div class="section" id="id74">
<h3>Known Issues<a class="headerlink" href="#id74" title="Permalink to this headline">🔗</a></h3>
<ul>
<li><p class="first"><a class="reference external" href="https://issues.cask.co/browse/CDAP-7175">CDAP-7175</a> - If you are upgrading an authorization-enabled CDAP instance,
you will need to give the <em>cdap</em> user <em>ADMIN</em> privileges on all existing CDAP
namespaces. See the <a class="reference external" href="http://docs.cask.co/cdap/3.5.1/en/admin-manual/upgrading/index.html#upgrading-index">管理手册: Upgrading</a>
for your distribution for details.</p>
</li>
<li><p class="first"><a class="reference external" href="https://issues.cask.co/browse/CDAP-7179">CDAP-7179</a> - In CDAP 3.5.0, new <code class="docutils literal notranslate"><span class="pre">kafka.server.*</span></code> properties replace older
properties such as <code class="docutils literal notranslate"><span class="pre">kafka.log.dir</span></code>, as described in the <a class="reference external" href="http://docs.cask.co/cdap/3.5.1/en/admin-manual/appendices/cdap-site.html#appendix-cdap-default-deprecated-properties">管理手册:
Appendices: cdap-site.xml</a>.</p>
<p><strong>If you are upgrading from CDAP 3.4.x to 3.5.x</strong> and you have set a value for
<code class="docutils literal notranslate"><span class="pre">kafka.log.dir</span></code> by using Cloudera Manager’s <a class="reference external" href="http://docs.cask.co/cdap/3.5.1/en/admin-manual/installation/cloudera.html#cloudera-installation-add-service-wizard-configuration">safety-valve mechanism</a>,
you need to change to the new
property <code class="docutils literal notranslate"><span class="pre">kafka.server.log.dirs</span></code>, as the deprecated <code class="docutils literal notranslate"><span class="pre">kafka.log.dir</span></code> is being ignored
in favor of the new property. If you don’t, your custom value will be replaced with the
default value.</p>
</li>
</ul>
</div>
<div class="section" id="id75">
<h3>Improvements<a class="headerlink" href="#id75" title="Permalink to this headline">🔗</a></h3>
<ul>
<li><p class="first"><a class="reference external" href="https://issues.cask.co/browse/CDAP-7192">CDAP-7192</a> - Added the ability to specify an announce address and port for
the <code class="docutils literal notranslate"><span class="pre">appfabric</span></code> and <code class="docutils literal notranslate"><span class="pre">dataset</span></code> services.</p>
<p>Deprecated the properties <code class="docutils literal notranslate"><span class="pre">app.bind.address</span></code> and <code class="docutils literal notranslate"><span class="pre">dataset.service.bind.address</span></code>,
replacing them with <code class="docutils literal notranslate"><span class="pre">master.services.bind.address</span></code> as the bind address for master
services.</p>
<p>Added the properties <code class="docutils literal notranslate"><span class="pre">master.services.announce.address</span></code>, <code class="docutils literal notranslate"><span class="pre">app.announce.port</span></code>, and
<code class="docutils literal notranslate"><span class="pre">dataset.service.announce.port</span></code> for use as announce addresses that are different from
the bind address.</p>
</li>
<li><p class="first"><a class="reference external" href="https://issues.cask.co/browse/CDAP-7240">CDAP-7240</a> - Upgraded the version of <code class="docutils literal notranslate"><span class="pre">netty-http</span></code> used in CDAP to version
0.15, resolving a problem with a NullPointerException being logged on the closing of a
network connection.</p>
</li>
<li><p class="first"><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-578">HYDRATOR-578</a> - Snapshot sinks now allow users to specify a property
<code class="docutils literal notranslate"><span class="pre">cleanPartitionsOlderThan</span></code> that cleans up any snapshots older than “x” days.</p>
</li>
</ul>
</div>
<div class="section" id="id76">
<h3>Bug Fixes<a class="headerlink" href="#id76" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-6215">CDAP-6215</a> - PartitionConsumer appropriately drops partitions that have
been deleted from a corresponding PartitionedFileSet.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-6404">CDAP-6404</a> - Fixed an issue with searching for an entity in Cask Tracker by
metadata after a tag with the same prefix has been removed.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7138">CDAP-7138</a> - Fixed a problem with duplicate logs showing for a running program.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7175">CDAP-7175</a> - Fixed a bug in the upgrade tool to allow it to run on a CDAP
instance with authorization enabled.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7178">CDAP-7178</a> - Fixed an issue with uploading an application JAR or file to a
stream through the CDAP UI.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7187">CDAP-7187</a> - Fixed a problem with the property
<code class="docutils literal notranslate"><span class="pre">dataset.service.bind.address</span></code> having no effect.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7199">CDAP-7199</a> - Corrected errors in the documentation to correctly show how to
set the schema on an existing table.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7204">CDAP-7204</a> - Lineage information is now returned for deleted datasets.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7222">CDAP-7222</a> - Fixed a problem with being unable to delete a namespace if a
configured keytab file doesn’t exist.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7235">CDAP-7235</a> - Fixed a problem with a NullPointerException when the CDAP UI fetches a log.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7237">CDAP-7237</a> - Prevented accidental grant of additional actions to a user as
part of a grant operation when using Apache Sentry as the authorization provider.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7248">CDAP-7248</a> - Fixed a problem with the FileBatchSource not working with Azure Blob Storage.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7249">CDAP-7249</a> - Fixed a problem with CDAP Explore using Tez on Azure HDInsight.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-912">HYDRATOR-912</a> - Fixed an issue where the Joiner plugin was failing in
Hydrator pipelines executing in a Spark environment.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-922">HYDRATOR-922</a> - Fixed a bug that caused the Database Source, Joiner,
GroupByAggregate, and Deduplicate plugins to fail on certain versions of Spark.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-932">HYDRATOR-932</a> - Fixed an error in the documentation of the HDFS Source and
Sink with respect to the alias under high-availability.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/TRACKER-217">TRACKER-217</a> - Fixed an issue preventing the adding of additional tags
after an existing tag had been deleted.</li>
</ul>
</div>
</div>
<div class="section" id="release-3-5-0">
<h2><a class="reference external" href="http://docs.cask.co/cdap/3.5.0/index.html">Release 3.5.0</a><a class="headerlink" href="#release-3-5-0" title="Permalink to this headline">🔗</a></h2>
<div class="section" id="id77">
<h3>Known Issues<a class="headerlink" href="#id77" title="Permalink to this headline">🔗</a></h3>
<ul>
<li><p class="first"><a class="reference external" href="https://issues.cask.co/browse/CDAP-7179">CDAP-7179</a> - In CDAP 3.5.0, new <code class="docutils literal notranslate"><span class="pre">kafka.server.*</span></code> properties replace older
properties such as <code class="docutils literal notranslate"><span class="pre">kafka.log.dir</span></code>, as described in the <a class="reference external" href="http://docs.cask.co/cdap/3.5.0/en/admin-manual/appendices/cdap-site.html#appendix-cdap-default-deprecated-properties">管理手册:
Appendices: cdap-site.xml</a>.</p>
<p><strong>If you are upgrading from CDAP 3.4.x to 3.5.x,</strong> and you have set a value for
<code class="docutils literal notranslate"><span class="pre">kafka.log.dir</span></code> by using Cloudera Manager’s <a class="reference external" href="http://docs.cask.co/cdap/3.5.0/en/admin-manual/installation/cloudera.html#cloudera-installation-add-service-wizard-configuration">safety-valve mechanism</a>,
you need to change to the
new property <code class="docutils literal notranslate"><span class="pre">kafka.server.log.dirs</span></code>, as the deprecated <code class="docutils literal notranslate"><span class="pre">kafka.log.dir</span></code> is being
ignored in favor of the new property. If you don’t, your custom value will be replaced
with the default value.</p>
</li>
</ul>
</div>
<div class="section" id="id78">
<h3>API Changes<a class="headerlink" href="#id78" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4860">CDAP-4860</a> - Introduced an “available” (<code class="docutils literal notranslate"><span class="pre">/available</span></code>) endpoint for
Services to check their availability.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-5279">CDAP-5279</a> - The <code class="docutils literal notranslate"><span class="pre">beforeSubmit</span></code> and <code class="docutils literal notranslate"><span class="pre">onFinish</span></code> methods of the MapReduce
and Spark APIs have been deprecated. Changes to the API include:<ol class="arabic">
<li><code class="docutils literal notranslate"><span class="pre">AbstractMapReduce</span></code> and <code class="docutils literal notranslate"><span class="pre">AbstractSpark</span></code> now implement <code class="docutils literal notranslate"><span class="pre">ProgramLifeCycle</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">AbstractMapReduce</span></code> and <code class="docutils literal notranslate"><span class="pre">AbstractSpark</span></code> now have a <code class="docutils literal notranslate"><span class="pre">final</span>
<span class="pre">initialize(context)</span></code> method</li>
<li><code class="docutils literal notranslate"><span class="pre">AbstractMapReduce</span></code> and <code class="docutils literal notranslate"><span class="pre">AbstractSpark</span></code> now have a <code class="docutils literal notranslate"><span class="pre">protected</span> <span class="pre">initialize()</span></code>
method default implementation of which will call <code class="docutils literal notranslate"><span class="pre">beforeSubmit()</span></code></li>
<li>User programs will override the no-arg initialize method</li>
<li>Driver will call both versions of the initialize method</li>
</ol>
</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-6150">CDAP-6150</a> - The <code class="docutils literal notranslate"><span class="pre">isSuccessful()</span></code> method of the WorkflowContext is
replaced by the <code class="docutils literal notranslate"><span class="pre">getState()</span></code> method, which returns the state of the workflow.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-6930">CDAP-6930</a> - <strong>Incompatible Change:</strong> Updated the “cdap-clients” to throw
<code class="docutils literal notranslate"><span class="pre">UnauthorizedException</span></code> when an operation returns <code class="docutils literal notranslate"><span class="pre">403</span> <span class="pre">-</span> <span class="pre">Forbidden</span></code> from CDAP. Users
of “cdap-clients” may need to update their code to handle these exceptions.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/TRACKER-21">TRACKER-21</a> - Renamed the AuditLog service to the TrackerService.</li>
</ul>
</div>
<div class="section" id="id79">
<h3>New Features<a class="headerlink" href="#id79" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2963">CDAP-2963</a> - All HBase Tables created through CDAP will now have a key
<code class="docutils literal notranslate"><span class="pre">cdap.version</span></code> in the <code class="docutils literal notranslate"><span class="pre">HTableDescriptor</span></code>.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3368">CDAP-3368</a> - Add location for <code class="docutils literal notranslate"><span class="pre">cdap</span> <span class="pre">cli</span></code> to PATH in distributed CDAP
packages.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3890">CDAP-3890</a> - Improved performance of the Dataset Service.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4106">CDAP-4106</a> - Created pre-defined alert definitions in the CDAP Ambari
Service.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4107">CDAP-4107</a> - Support for HA CDAP installations in the CDAP Ambari Service.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4109">CDAP-4109</a> - Support for Kerberos-enabled clusters via the CDAP Ambari service.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4110">CDAP-4110</a> - CDAP Auth Server is now supported in the CDAP Ambari Service
on Ambari clusters which have Kerberos enabled.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4288">CDAP-4288</a> - Added an authorization extension backed by Apache Sentry to
enforce authorization on CDAP entities.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4913">CDAP-4913</a> - Added a way to cache authorization policies so every
authorization enforcement request does not have to make a remote call. Caching is
configurable—it can be enabled by setting security.authorization.cache.enabled to
true. TTL for cache entries (<code class="docutils literal notranslate"><span class="pre">security.authorization.cache.ttl.secs</span></code>) as well as refresh
interval (<code class="docutils literal notranslate"><span class="pre">security.authorization.cache.refresh.interval.secs</span></code>) is also configurable.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-5740">CDAP-5740</a> - Provided access to <code class="docutils literal notranslate"><span class="pre">Partitioner</span></code> and <code class="docutils literal notranslate"><span class="pre">Comparator</span></code> classes
to the <code class="docutils literal notranslate"><span class="pre">MapReduceTaskContext</span></code> by implementing <code class="docutils literal notranslate"><span class="pre">ProgramLifeCycle</span></code>.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-5770">CDAP-5770</a> - Provided setting of YARN container resources requirements for all
program types via preferences and runtime arguments.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-6062">CDAP-6062</a> - Added protection for a partition of a file set from being
deleted while a query is reading the partition.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-6153">CDAP-6153</a> - CDAP namespaces can now be mapped to custom namespaces in
storage providers. While creating a namespace, users can specify the Filesystem directory,
HBase namespace and Hive database for that namespace. These settings cannot be changed
once the namespace has been successfully created.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-6168">CDAP-6168</a> - Enable authorization, lineage, and audit log at the data
operation level for all Datasets.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-6174">CDAP-6174</a> - Addes a new log viewer across CDAP, Cask Hydrator, and Cask
Tracker, wherever appropriate. Provides easier navigation and debugging functionality for
logs of different entities.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-6235">CDAP-6235</a> - Added an indicator in the UI of the CDAP mode (distributed or
standalone, secure or insecure).</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-6393">CDAP-6393</a> - Added authorization to the Secure Key HTTP RESTful APIs. To
create a secure key, a user needs WRITE privilege on the namespace in which the key is
being created. Users can only view secure keys that they have access to. To delete a key,
ADMIN privilege is required.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-6456">CDAP-6456</a> - Exposed the secure store APIs to Programs.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-6516">CDAP-6516</a> - Added authorization for listing and viewing CDAP entities.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7002">CDAP-7002</a> - Fixed an issue where the UI would ignore the configured port
when connecting to the CDAP Router.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-156">HYDRATOR-156</a> - Added an alpha feature: Hydrator Data Pipeline preview
(CDAP SDK only).</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-162">HYDRATOR-162</a> - Added support for executing custom actions in the Cask
Hydrator pipelines.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-168">HYDRATOR-168</a> - Re-organized the bottom panel in Cask Hydrator to be
in-context. Pipeline-level information is moved to a top panel and plugin-level
information is moved to a modal dialog.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-379">HYDRATOR-379</a> - Re-organized the left panel in Cask Hydrator studio view to
have a maximum of four categories of plugin types: Source, Transform, Sink, and Actions.
All other types are consolidated into one of these types.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-501">HYDRATOR-501</a> - Implemented the Value Mapper plugin for Cask Hydrator
plugins. This is a type of transform that maps string values of a field in the input
record to another value.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-502">HYDRATOR-502</a> - Added the XML Parser Transform plugin to Cask Hydrator
plugins. This plugin uses XPath to extract fields from a complex XML Event. It is
generally used in conjunction with the XML Reader Source Plugin.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-503">HYDRATOR-503</a> - Added the XML Reader Source Plugin to Cask Hydrator
plugins. This plugin allows users to read XML files stored on HDFS.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-506">HYDRATOR-506</a> - Implemented the Cask Hydrator plugin for Row Denormalizer
aggregator. This plugin converts raw data into de-normalized data based on a key column.
De-normalized data can be easier and faster to query.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-507">HYDRATOR-507</a> - Added the Cobol Copybook source plugin to Cask Hydrator
plugins. This source plugin allows users to read and process mainframe files defined using
COBOL Copybook.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-514">HYDRATOR-514</a> - Added the Excel Reader Source plugin to Cask Hydrator
Plugins. This plugin provides the ability to read data from one or more Excel file(s).</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-629">HYDRATOR-629</a> - Adds macros to pipeline plugin configurations. This allows
users to set macros for plugin properties which can be provided as runtime arguments while
scheduling and running the pipeline.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-634">HYDRATOR-634</a> - Adds a new Run Configuration player for published pipeline
views. This allows users to set runtime arguments while scheduling or running a pipeline.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-685">HYDRATOR-685</a> - Added a Twitter source for Spark Streaming pipelines.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/TRACKER-96">TRACKER-96</a> - Added the ability to edit user properties for a dataset
directly in Cask Tracker.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/TRACKER-98">TRACKER-98</a> - Added the Cask Tracker Meter to measure how active a dataset
is in a cluster on a scale of zero to 100.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/TRACKER-100">TRACKER-100</a> - Added the ability to add, remove, and manage a common
dictionary of Preferred Tags in Cask Tracker and apply them to datasets.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/TRACKER-104">TRACKER-104</a> - Added the ability to preview data directly in the Cask
Tracker UI.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/TRACKER-105">TRACKER-105</a> - Added the ability to view usage metrics about datasets in
Cask Tracker. Users can view how many applications and programs are accessing each dataset
using service endpoints and the Tracker UI.</li>
</ul>
</div>
<div class="section" id="changes">
<h3>Changes<a class="headerlink" href="#changes" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-5263">CDAP-5263</a> - The “CDAP Applications” section in the documentation has been
split into two separate sections under “CDAP Extensions”: “Cask Hydrator” and “Cask
Tracker”.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-5833">CDAP-5833</a> - Eliminated some misleading warnings in the Purchase example.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-6143">CDAP-6143</a> - Added metadata tag for the local datasets.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-6596">CDAP-6596</a> - CDAP Security Extensions are packaged with CDAP Master
packages and CDAP Parcel.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-527">HYDRATOR-527</a> - The Script Transform (previously deprecated) has been
removed, and is replaced with the JavaScript Transform.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-528">HYDRATOR-528</a> - Secure Store APIs in Hydrator Actions are now exposed.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-649">HYDRATOR-649</a> - A widget textbox can have a configurable placeholder.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-653">HYDRATOR-653</a> - Additional Custom Action Hydrator Plugins have been added.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-682">HYDRATOR-682</a> - The directory containing the Spark Streaming Hydrator
plugins has been renamed from <code class="docutils literal notranslate"><span class="pre">batch.spark</span></code> to <code class="docutils literal notranslate"><span class="pre">spark</span></code>.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/TRACKER-155">TRACKER-155</a> - An upgrade process has been added to the UI for Cask
Tracker.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-886">CDAP-886</a>, <a class="reference external" href="https://issues.cask.co/browse/CDAP-882">CDAP-882</a> - Access to CDAP Streams via the RESTful
API, CDAP-CLI, or programmatic API can be authorized through the Security Authorization
feature.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-888">CDAP-888</a>, <a class="reference external" href="https://issues.cask.co/browse/CDAP-882">CDAP-882</a> - Enforced authorization in Dataset
RESTful APIs. Dataset modules, types, and instances are now governed by authorization
policies.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-5691">CDAP-5691</a>, <a class="reference external" href="https://issues.cask.co/browse/CDAP-5685">CDAP-5685</a> - Improved the performance of the
Dataset Service, with server-side caching of <code class="docutils literal notranslate"><span class="pre">getDataset()</span></code> in DatasetService.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-6154">CDAP-6154</a>, <a class="reference external" href="https://issues.cask.co/browse/CDAP-6153">CDAP-6153</a> - CDAP Namespaces can now use an
existing custom HDFS directory. The custom HDFS directory, whose creation/deletion is
managed by the user, can be specified during the creation of a CDAP Namespace as part of
its configuration.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-6155">CDAP-6155</a>, <a class="reference external" href="https://issues.cask.co/browse/CDAP-6153">CDAP-6153</a> - CDAP Namespaces can now use an
existing custom HBase namespace. The custom HBase namespace, whose creation/deletion is
managed by the user, can be specified during the creation of a CDAP Namespace as part of
its configuration.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-6156">CDAP-6156</a>, <a class="reference external" href="https://issues.cask.co/browse/CDAP-6153">CDAP-6153</a> - CDAP Namespaces can now use an
existing custom Hive database. The custom Hive database, whose creation/deletion is
managed by the user, can be specified during the creation of a CDAP Namespace as part of
its configuration.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-6158">CDAP-6158</a>, <a class="reference external" href="https://issues.cask.co/browse/CDAP-6157">CDAP-6157</a> - Added support for accessing
(read/write) dataset across namespaces in CDAP Spark and MapReduce programs.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-6159">CDAP-6159</a>, <a class="reference external" href="https://issues.cask.co/browse/CDAP-6157">CDAP-6157</a> - Added support for accessing streams
(read only) across namespaces in CDAP Spark and MapReduce programs.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-174">HYDRATOR-174</a>, <a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-157">HYDRATOR-157</a> - Refactored the Spark engine in
data pipelines to run all non-action pipeline stages in a single Spark program.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-175">HYDRATOR-175</a>, <a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-157">HYDRATOR-157</a> - Added a streaming pipeline type
(the Data Streams artifact) to Cask Hydrator for realtime pipelines run using Spark
Streaming.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-177">HYDRATOR-177</a>, <a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-157">HYDRATOR-157</a> - Added a Kafka source for
streaming pipelines.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-178">HYDRATOR-178</a>, <a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-157">HYDRATOR-157</a> - Added a window plugin to Cask
Hydrator that enables the creation of sliding windows in a streaming pipeline.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-181">HYDRATOR-181</a>, <a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-158">HYDRATOR-158</a> - Added an experimental feature
in the CDAP SDK which allows users to preview the Hydrator pipelines.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-182">HYDRATOR-182</a> - Hydrator MapReduce or Spark jobs now support multiple
inputs. This will enable more efficient physical workflow generation due to the reduction
in the number of MapReduce or Spark programs required for a logical pipeline.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-165">HYDRATOR-165</a> - Support multiple sources as input to a stage.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-748">HYDRATOR-748</a> - Re-organizes batch pipeline settings to a top panel to
schedule a batch pipeline, add post-run actions and set pipeline resources and the engine
used.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-712">HYDRATOR-712</a> - Added to Cask Hydrator a Batch Pipeline Configuration
Schedule.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/TRACKER-108">TRACKER-108</a>, <a class="reference external" href="https://issues.cask.co/browse/TRACKER-98">TRACKER-98</a> - Adds to Cask Tracker a tracker
meter widget in the UI, including search results page and details page. It displays a
metric that determines the ‘truthfulness’ of a dataset/stream being used in CDAP or Cask
Hydrator.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/TRACKER-109">TRACKER-109</a>, <a class="reference external" href="https://issues.cask.co/browse/TRACKER-100">TRACKER-100</a> - Adds a separate section for tags
in Cask Tracker. This lists all available tags in CDAP.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/TRACKER-149">TRACKER-149</a>, <a class="reference external" href="https://issues.cask.co/browse/TRACKER-105">TRACKER-105</a> - Adds a histogram for audit log in
Cask Tracker for easier visualization of usage of a dataset.</li>
</ul>
</div>
<div class="section" id="id80">
<h3>Improvements<a class="headerlink" href="#id80" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-1545">CDAP-1545</a> - Created a Docker-specific ENTRYPOINT script to support passing
arguments.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4065">CDAP-4065</a> - Improved the way that MapReduce failures are reported.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4775">CDAP-4775</a> - Warns if either the app-fabric or router bind addresses are
configured with a loopback address.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-5000">CDAP-5000</a> - The number of containers for the CDAP Explore service is no
longer configurable and will be ignored upon specification. It will always be set to one
(1).</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-5336">CDAP-5336</a> - Now publishing <code class="docutils literal notranslate"><span class="pre">stdout</span></code> and <code class="docutils literal notranslate"><span class="pre">stderr</span></code> logs for MapReduce
containers to CDAP.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-5601">CDAP-5601</a> - Allowing the setting of batch size for flowlet process methods
via preferences and runtime arguments.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-5794">CDAP-5794</a> - Added support for long-running Spark jobs in a
Kerberos-enabled cluster.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-5874">CDAP-5874</a> - Added support for starting extensions in distributed mode.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-5959">CDAP-5959</a> - Setting the <code class="docutils literal notranslate"><span class="pre">JAVA_LIBRARY_PATH</span></code> now causes CDAP Master to
load Hadoop native libraries at startup.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-5969">CDAP-5969</a> - CDAP Upgrade tasks are now available in the CDAP Ambari
Service.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-6034">CDAP-6034</a> - CDAP’s Tephra dependency has been changed to depend on the
<a class="reference external" href="http://tephra.incubator.apache.org">Apache Incubator Tephra project</a>.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-6206">CDAP-6206</a> - Improved the error message given on application deployment
failure due to a missing Spark library.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-6216">CDAP-6216</a> - Added support in the log API for field suppression in JSON
format.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-6246">CDAP-6246</a> - Added the ability to specify a CDAP Master’s temporary
directory.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-6276">CDAP-6276</a> - Introduced new experimental dataset APIs for updating a
dataset’s properties.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-6327">CDAP-6327</a> - Allowed specifying individual Java heap sizes for Java
services in <code class="docutils literal notranslate"><span class="pre">cdap-env.sh</span></code>.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-6350">CDAP-6350</a> - Declared startup script contents as read-only to prevent them
from being overridden by a user in <code class="docutils literal notranslate"><span class="pre">cdap-env.sh</span></code>.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-6361">CDAP-6361</a> - Added “Quick Links” for the CDAP UI, Cask Hydrator, and Cask
Tracker in the Ambari 2.3+ UI.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-6362">CDAP-6362</a> - Added support for CDAP services over SSL in Ambari.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-6363">CDAP-6363</a> - Provided service dependencies for Ambari (requires Ambari
2.2+).</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-6384">CDAP-6384</a> - Updated the Standalone CDAP VM version of IntelliJ IDE to
2016.1.3.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-6573">CDAP-6573</a> - Added a tool that allows bringing Hive in-sync with the
partitions of a (time-)partitioned fileset.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-6880">CDAP-6880</a> - Users can now configure timeouts for internal HTTP connections
and reads in <code class="docutils literal notranslate"><span class="pre">cdap-site.xml</span></code>. These are used for all internal HTTP calls.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-6901">CDAP-6901</a> - Added a bootstrap step for authorization in CDAP. As part of
this step:<ol class="arabic">
<li>The user that CDAP runs as now receives “admin” privileges on the CDAP instance, as
well as “all” privileges on the system namespace.</li>
<li>The list of users specified in the parameter <code class="docutils literal notranslate"><span class="pre">security.authorization.admin.users</span></code>
in cdap-site.xml receives “admin” privileges on the CDAP instance so that they can
create namespaces.</li>
</ol>
</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-6913">CDAP-6913</a> - Changed to use <code class="docutils literal notranslate"><span class="pre">YarnClient</span></code> instead of the YARN HTTP API to
fetch node reports.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7021">CDAP-7021</a> - Improved program launch performance to avoid large CPU spikes
when multiple programs are launched at the same time.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7046">CDAP-7046</a> - At configure time, <code class="docutils literal notranslate"><span class="pre">containsMacro(.)</span></code> on plugin properties
that were provided macro syntax will return true. At runtime, all properties will have
<code class="docutils literal notranslate"><span class="pre">containsMacro(.)</span></code> return false.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-219">HYDRATOR-219</a> - Added a new editor for complex schema in the Cask Hydrator
UI.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-244">HYDRATOR-244</a> - Added support for macros in plugins. This allows Cask
Hydrator plugin fields to accept macros.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-289">HYDRATOR-289</a> - Added support to join data from multiple sources in Cask
Hydrator.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-392">HYDRATOR-392</a> - Enhanced the Cask Hydrator upgrade tool to upgrade 3.4.x
pipelines to 3.5.x pipelines.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-560">HYDRATOR-560</a> - The plugins NaiveBayesTrainer and NaiveBayesClassifier now
have an optional configurable <code class="docutils literal notranslate"><span class="pre">features</span></code> property. If specified as <code class="docutils literal notranslate"><span class="pre">none</span></code>, <code class="docutils literal notranslate"><span class="pre">100</span></code> is
used as the number of features.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-578">HYDRATOR-578</a> - Snapshot sinks now allow users to specify a property
<code class="docutils literal notranslate"><span class="pre">cleanPartitionsOlderThan</span></code> that cleans up any snapshots older than <code class="docutils literal notranslate"><span class="pre">x</span></code> days.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-606">HYDRATOR-606</a> - Changed the DBSource plugin to override user-specified
output schema.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-607">HYDRATOR-607</a> - Fixed an issue that prevented TPFS sources and sinks
created by Hydrator pipelines from being used as either input or output for MapReduce or
Spark.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-686">HYDRATOR-686</a> - Many existing Hydrator batch and spark plugins now have
macro-enabled properties, as specified in their reference documentation.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-713">HYDRATOR-713</a> - Added Encryptor and Decryptor plugins to Cask Hydrator that
can encrypt or decrypt record fields.</li>
</ul>
</div>
<div class="section" id="id81">
<h3>Bug Fixes<a class="headerlink" href="#id81" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2501">CDAP-2501</a> - The CDAP Router and UI no longer need to be colocated using
Cloudera Manager.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3131">CDAP-3131</a> - Running the endpoint of the Program Lifecycle RESTful API now
returns <code class="docutils literal notranslate"><span class="pre">404</span></code> instead of an empty list if a specified application is not found.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3732">CDAP-3732</a> - Fixed an issue where deploying an application was trying to
enable CDAP Explore on system tables.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3750">CDAP-3750</a> - Datasets that use reserved Hive keywords will now have their
column names properly escaped when executing Hive DDL commands.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4007">CDAP-4007</a> - Fixed an issue when running multiple unit tests in the same
JVM.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4434">CDAP-4434</a> - CDAP startup scripts return success (exit 0) if calling a
service that is already running.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-5135">CDAP-5135</a> - Fixed an issue where the status of a program that was killed
through YARN showed in CDAP as having been completed successfully.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-5291">CDAP-5291</a> - Fixed a problem in the fit-to-screen functionality of flow
diagrams.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-5536">CDAP-5536</a> - Fixed a problem with users putting back a partition to
PartitionConsumer without processing it.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-5643">CDAP-5643</a> - Fixed certain test cases to not depend on <code class="docutils literal notranslate"><span class="pre">US</span></code> as the system
locale.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-5676">CDAP-5676</a> - Upgraded the Hive version used by the CDAP SDK to Hive-1.2.1
in order to pick up a fix for parquet tables.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-5875">CDAP-5875</a> - Require Spark on clusters configured for Hive on Spark and
CDAP Explore service.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-5882">CDAP-5882</a> - Removed conditional restart on distributed CDAP package
upgrades.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-6026">CDAP-6026</a> - Fixed an issue where an exception thrown in the initialize
method of the Workflow was causing the Workflow container not to be terminated.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-6035">CDAP-6035</a> - Fixed a problem with correctly setting the context classloader
for the Workflow <code class="docutils literal notranslate"><span class="pre">initialize()</span></code> and <code class="docutils literal notranslate"><span class="pre">destroy(</span></code>) methods, to provide a consistent
classloading behavior across all program types.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-6045">CDAP-6045</a> - Fixed an issue where application deployment was failing on
Windows because of a colon (“:”) character in the filename.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-6052">CDAP-6052</a> - Fixed a bug that prevented the setting of <code class="docutils literal notranslate"><span class="pre">local.data.dir</span></code>
in <code class="docutils literal notranslate"><span class="pre">cdap-site.xml</span></code> to an absolute path.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-6109">CDAP-6109</a> - Fixed a NullPointerException issue in Spark when saving RDD to
a PartitionedFileSet dataset.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-6115">CDAP-6115</a> - Fixed a bug in the Flow system where usage of the primitive
<code class="docutils literal notranslate"><span class="pre">byte</span></code>, <code class="docutils literal notranslate"><span class="pre">short</span></code>, or <code class="docutils literal notranslate"><span class="pre">char</span></code> types caused exceptions.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-6121">CDAP-6121</a> - Fixed a bug in Spark where using <code class="docutils literal notranslate"><span class="pre">&#64;UseDataset</span></code> caused a
NullPointerException.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-6127">CDAP-6127</a> - Fixed a bug not allowing the transaction service to bind to a
configurable port.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-6147">CDAP-6147</a> - Improved the error message in the authorization and lineage
clients when a <code class="docutils literal notranslate"><span class="pre">404</span></code> is returned from the server side.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-6170">CDAP-6170</a> - Fixed an issue that caused an error if an application or
program attempted to override input/output format properties that were already defined in
the dataset properties.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-6280">CDAP-6280</a> - Fixed a problem with allowing FileSets and PartitionedFileSets
to be tagged as explorable in the CDAP UI.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-6311">CDAP-6311</a> - Fixed a bug that the program run record was not correctly
reflected in CDAP if the corresponding YARN application failed to start.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-6378">CDAP-6378</a> - Fixed the classpath of the MapReduce program launched by CDAP
to include the CDAP classes before the Apache Twill classes.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-6386">CDAP-6386</a> - Fixed an issue where updating the properties of a dataset
deleted all of its partitions in Hive.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-6452">CDAP-6452</a> - Add a check for the environment variable
<code class="docutils literal notranslate"><span class="pre">CDAP_UI_COMPRESSION_ENABLED</span></code> to disable UI compression.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-6455">CDAP-6455</a> - Fixed the classpath of a MapReduce program launched by the
explore service to include the <code class="docutils literal notranslate"><span class="pre">cdap-common.jar</span></code> at the beginning.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-6486">CDAP-6486</a> - Fixed an issue that caused a Zookeeper watch to leak memory
every time a program was started.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-6510">CDAP-6510</a> - Fixed an issue where the ExploreService was attempting (with
no effect except for a slow down) to run the upgrade procedure for all explorable
datasets.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-6515">CDAP-6515</a> - Fixed classloading issues related to using Guava’s
<code class="docutils literal notranslate"><span class="pre">Optional</span></code> class in Spark, allowing programs to perform left-outer and full-outer joins
on RDDs.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-6524">CDAP-6524</a> - Plugins now support the <code class="docutils literal notranslate"><span class="pre">char</span></code> primitive as a property type.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-6643">CDAP-6643</a> - Fixed an issue that caused massive log messages when there was
an underlying HDFS issues.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-6783">CDAP-6783</a> - Fixed the classpath ordering in Spark to load the classes from
<code class="docutils literal notranslate"><span class="pre">cdap-common</span></code> first.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-6829">CDAP-6829</a> - Fixed issues that prevented the Log Saver from performing
cleanup when metadata is present for a non-existing file.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-6852">CDAP-6852</a> - Fixed issues that makes the Log Saver more resilient to errors
while checkpointing.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-6860">CDAP-6860</a> - Improved performance in cube datasets when querying for more
than one measure in a query. This will also improve metrics query performance.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-6929">CDAP-6929</a> - Logs from Spark driver and executors are now collected.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-6935">CDAP-6935</a> - Fix a bug where the live-info endpoint was not working for
Workflows, MapReduce, Worker, and Spark.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-6939">CDAP-6939</a> - Added support in the CDAP UI for Google Chrome releases prior
to version 44.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7026">CDAP-7026</a> - Upon namespace creation, all privileges are granted to both
the user who created the namespace as well as the user that programs will run as in the
new namespace.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7066">CDAP-7066</a> - Restart of system services now kills containers if the
containers are unresponsive so as to not leave stray containers.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7082">CDAP-7082</a> - Removed bundling the parquet JAR from the <code class="docutils literal notranslate"><span class="pre">com.twitter</span></code>
package with CDAP Master.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-7128">CDAP-7128</a> - Fixed a bug on changing the number of Worker instances in CDAP
Distributed mode.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-47">HYDRATOR-47</a> - The DBSource plugin now casts <code class="docutils literal notranslate"><span class="pre">TINYINT</span></code> and <code class="docutils literal notranslate"><span class="pre">SMALLINT</span></code>
to <code class="docutils literal notranslate"><span class="pre">INT</span></code> type correctly.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-54">HYDRATOR-54</a> - The Validator UI configuration is now preserved in a cloned
pipeline.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-80">HYDRATOR-80</a> - Fixed an issue where the configuration of the FileSource was
failing while setting the properties for the FileInputFormat.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-133">HYDRATOR-133</a> - HDFSSink can now be used alongside other sinks in a
Hydrator pipeline.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-149">HYDRATOR-149</a> - Removes the dependency of using labels from plugins in
pipelines being imported in UI. Any pipeline configuration publishable from the CDAP-CLI
or the Artifact RESTful HTTP API should now be publishable from UI.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-398">HYDRATOR-398</a> - Adds the ability to view properties of plugins in pipelines
created in older versions of Cask Hydrator.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-438">HYDRATOR-438</a> - Fixed the Hydrator CSVParser plugin so that a nullable
field is only set to null if the parsed value is an empty string and the field is not
either a string or nullable string type.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-451">HYDRATOR-451</a> - The CSVParser plugin now supports accepting a nullable
string as a field to parse. If the field is null, all other fields are propagated and
those that would otherwise be parsed by the CSVParser are set to null.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-459">HYDRATOR-459</a> - Fixed a bug causing the UPPER to lower transform not being
applied to all columns correctly for DBSink.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-705">HYDRATOR-705</a> - Fixed an issue with record serialization for non-ASCII
values in the shuffle phase of Hydrator pipelines.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/HYDRATOR-790">HYDRATOR-790</a> - Release CDAP 3.4.0 introduced infinite-scroll for the input
and output schemas; the version used (1.2.2) of the infinite scroll component had
performance issues. The version of the infinite scroll component used has been downgraded
to restore the performance in Hydrator views.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/TRACKER-42">TRACKER-42</a> - Fixed integrating the navigator app in the Cask Tracker UI.
The POST body request that was sent while deploying the navigator app was using an
older, deprecated property (UI was using <code class="docutils literal notranslate"><span class="pre">metadataKafkaConfig</span></code> instead of
<code class="docutils literal notranslate"><span class="pre">auditKafkaConfig</span></code>). This should enable using the navigator app in the Cask Tracker UI.</li>
</ul>
</div>
</div>
<div class="section" id="release-3-4-1">
<h2><a class="reference external" href="http://docs.cask.co/cdap/3.4.1/index.html">Release 3.4.1</a><a class="headerlink" href="#release-3-4-1" title="Permalink to this headline">🔗</a></h2>
<div class="section" id="id82">
<h3>Bug Fixes<a class="headerlink" href="#id82" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4388">CDAP-4388</a> - Fixed a race
condition bug in ResourceCoordinator that prevented performing partition
assignment in the correct order. It affects the metrics processor and
stream coordinator.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-5855">CDAP-5855</a> - Avoid the
cancellation of delegation tokens upon completion of Explore-launched
MapReduce and Spark jobs, as these delegation tokens are shared by CDAP
system services.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-5868">CDAP-5868</a> - Removed
‘SNAPSHOT’ from the artifact version of apps created by default by the CDAP UI.
This fixes deploying Cask Tracker and Navigator apps, enabling Cask Tracker
from the CDAP UI.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-5884">CDAP-5884</a> - Fixed a bug
that caused SDK builds to fail when using 3.3.x versions of maven.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-5887">CDAP-5887</a> - Fixed the
Hydrator upgrade tool to correctly write out pipeline configs that
failed to upgrade.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-5889">CDAP-5889</a> - The CDAP
Standalone now deploys and starts the Cask Tracker app in the default
namespace if the Tracker artifact is present.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-5898">CDAP-5898</a> - Shutdown
external processes started by CDAP (Zookeeper and Kafka) when there is
an error during either startup or shutdown of CDAP.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-5907">CDAP-5907</a> - Fixed an
issue where parsing of an AVRO schema was failing when it included
optional fields such as ‘doc’ or ‘default’.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-5947">CDAP-5947</a> - Fixed a bug
in the BatchReadableRDD so that it won’t skip records when used by
DataFrame.</li>
</ul>
</div>
<div class="section" id="id83">
<h3>Known Issues<a class="headerlink" href="#id83" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li>After upgrading CDAP from a pre-3.0 version, any unprocessed metrics data in Kafka will
be lost and <em>WARN</em> log messages will be logged that tell about the inability to process
old data in the old format.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-797">CDAP-797</a> -
When running secure Hadoop clusters, debug logs from MapReduce programs are not
available.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-1007">CDAP-1007</a> -
If the Hive Metastore is restarted while the CDAP Explore Service is running, the
Explore Service remains alive, but becomes unusable. To correct, restart the CDAP Master—which will restart all services—as described under “Starting CDAP Services”
for your particular Hadoop distribution in the <a class="reference external" href="http://docs.cask.co/cdap/3.4.1/en/admin-manual/installation/index.html#installation-index">Installation documentation</a>.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-1587">CDAP-1587</a> -
CDAP internally creates tables in the “user” space that begin with the word
<code class="docutils literal notranslate"><span class="pre">&quot;system&quot;</span></code>. User datasets with names starting with <code class="docutils literal notranslate"><span class="pre">&quot;system&quot;</span></code> can conflict if they
were to match one of those names. To avoid this, do not start any datasets with the word
<code class="docutils literal notranslate"><span class="pre">&quot;system&quot;</span></code>.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2632">CDAP-2632</a> -
The application in the <a class="reference external" href="https://github.com/cdap-guides/cdap-kafka-ingest-guide/tree/release/cdap-3.0-compatible">cdap-kafka-ingest-guide</a>
does not run on Ubuntu 14.x as of CDAP 3.0.x.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2721">CDAP-2721</a> -
Metrics for <a class="reference external" href="http://docs.cask.co/cdap/3.4.1/en/developers-manual/building-blocks/datasets/fileset.html#datasets-fileset">FileSets</a>
can show zero values even if there is
data present, because FileSets do not emit metrics (<a class="reference external" href="https://issues.cask.co/browse/CDAP-587">CDAP-587</a>).</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2831">CDAP-2831</a> -
A workflow that is scheduled by time will not be run between the failure of the primary
master and the time that the secondary takes over. This scheduled run will not be
triggered at all.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2920">CDAP-2920</a> - Spark jobs on a
Kerberos-enabled CDAP cluster cannot run longer than the delegation token expiration.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2945">CDAP-2945</a> -
If the input partition filter for a PartitionedFileSet does not match any partitions,
MapReduce jobs can fail.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3000">CDAP-3000</a> -
The Workflow token is in an inconsistent state for nodes in a fork while the nodes of
the fork are still running. It becomes consistent after the join.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3221">CDAP-3221</a> -
When running in Standalone CDAP, if a MapReduce job fails repeatedly, then the SDK
hits an out-of-memory exception due to <code class="docutils literal notranslate"><span class="pre">perm</span> <span class="pre">gen</span></code>. The Standalone needs restarting at
this point.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3262">CDAP-3262</a> -
For Microsoft Windows, the Standalone CDAP scripts can fail when used with a JAVA_HOME
that is defined as a path with spaces in it. A workaround is to use a definition of
JAVA_HOME that does not include spaces, such as <code class="docutils literal notranslate"><span class="pre">C:\PROGRA~1\Java\jdk1.7.0_79\bin</span></code> or
<code class="docutils literal notranslate"><span class="pre">C:\ProgramData\Oracle\Java\javapath</span></code>.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3492">CDAP-3492</a> -
In the CDAP CLI, executing <code class="docutils literal notranslate"><span class="pre">select</span> <span class="pre">*</span></code> from a dataset with many fields generates an error.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3641">CDAP-3641</a> -
A RESTful API call to retrieve workflow statistics hangs if units (such as “s” for
seconds) are not provided as part of the query.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3750">CDAP-3750</a> -
If a table schema contains a field name that is a reserved word in the Hive DDL, <code class="docutils literal notranslate"><span class="pre">'enable</span> <span class="pre">explore'</span></code> fails.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-5900">CDAP-5900</a> - During the
upgrade to CDAP 3.4.1, publishing to Kafka is halted because the CDAP
Kafka service is not running. As a consequence, any applications that
sync to the CDAP metadata will become out-of-sync as changes to the
metadata made by the upgrade tool will not be published.</li>
</ul>
</div>
</div>
<div class="section" id="release-3-4-0">
<h2><a class="reference external" href="http://docs.cask.co/cdap/3.4.0/index.html">Release 3.4.0</a><a class="headerlink" href="#release-3-4-0" title="Permalink to this headline">🔗</a></h2>
<div class="section" id="id84">
<h3>API Changes<a class="headerlink" href="#id84" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-5082">CDAP-5082</a> - Added a new Spark Java and Scala API.</li>
</ul>
</div>
<div class="section" id="id85">
<h3>New Features<a class="headerlink" href="#id85" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-20">CDAP-20</a> - Removed dependency on the Guava
library from the <code class="docutils literal notranslate"><span class="pre">cdap-api</span></code> module. Applications are now free to use a Guava library
version of their choice.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3051">CDAP-3051</a> - Added capability for programs to
perform administrative dataset operations (create, update, truncate, drop).</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3854">CDAP-3854</a> - Added the capability to
configure Kafka topic for logs and notifications using the <code class="docutils literal notranslate"><span class="pre">cdap-site.xml</span></code>.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3980">CDAP-3980</a> - MapReduce programs submitted via CDAP
now support multiple configured inputs.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4807">CDAP-4807</a> - Added an ODBC 3.0 Driver for
CDAP Datasets for Windows-based applications that support an ODBC interface.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4970">CDAP-4970</a> - Added capability to fetch the
schema from a JDBC source specified for a Database plugin from inside Cask Hydrator.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-5011">CDAP-5011</a> - Added a CDAP extension <em>Cask Tracker</em>:
data discovery with metadata, audit, and lineage.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-5146">CDAP-5146</a> - Added a new Cask Hydrator
<code class="docutils literal notranslate"><span class="pre">batchaggregator</span></code> plugin type. An aggregator operates on a collection of records,
grouping them by a key and performing an aggregation on each group.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-5172">CDAP-5172</a> - Added support for
authorization extensions in CDAP. Extensions extend an <code class="docutils literal notranslate"><span class="pre">Authorizer</span></code> class and provide a
bundle jar containing all their required dependencies. This jar is then specified using
the property <code class="docutils literal notranslate"><span class="pre">security.authorization.extension.jar.path</span></code> in the <code class="docutils literal notranslate"><span class="pre">cdap-site.xml</span></code>.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-5191">CDAP-5191</a> - Added an <code class="docutils literal notranslate"><span class="pre">FTPBatchSource</span></code>
that can fetch data from an FTP server in a batch pipeline of Cask Hydrator.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-5205">CDAP-5205</a> - Added a global search across
all CDAP entities in the CDAP UI.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-5274">CDAP-5274</a> - The Cask Hydrator Studio now
includes the capability to configure a new type of pipeline, a “data pipeline” (beta
feature).</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-5360">CDAP-5360</a> - The CDAP UI now supports
<code class="docutils literal notranslate"><span class="pre">Sparksink</span></code> and <code class="docutils literal notranslate"><span class="pre">Sparkcompute</span></code> plugin types, included in a new “data pipeline”
artifact.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-5361">CDAP-5361</a> - Added a <code class="docutils literal notranslate"><span class="pre">SparkTransform</span></code>
plugin type, which allows the running of a Spark job that operates as a transform in an ETL
batch pipeline.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-5362">CDAP-5362</a> - Added a <code class="docutils literal notranslate"><span class="pre">SparkSink</span></code> plugin
type, which allows the running of a Spark job (such as machine learning) on the output of
an ETL batch pipeline.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-5392">CDAP-5392</a> - Added support for
<code class="docutils literal notranslate"><span class="pre">FormatSpecification</span></code> in Spark when consuming data from a stream.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-5446">CDAP-5446</a> - Added an example application
demonstrating the use of Spark Streaming with machine-learning and spam classifying.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-5504">CDAP-5504</a> - Added experimental support for
using Spark as an execution engine for CDAP Explore.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-5707">CDAP-5707</a> - Added support for using Tez as
an execution engine for CDAP Explore.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-5846">CDAP-5846</a> - Bundled <a class="reference external" href="https://nodejs.org/">Node.js</a> with the CDAP UI RPM and DEB packages and with the CDAP Parcels.</li>
</ul>
</div>
<div class="section" id="id86">
<h3>Improvements<a class="headerlink" href="#id86" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4071">CDAP-4071</a> - MapReduce programs can now be
configured to write metadata for each partition created using a <code class="docutils literal notranslate"><span class="pre">DynamicPartitioner</span></code>.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4117">CDAP-4117</a> - Fixed an issue of not using
the correct user account to access HDFS when submitting a YARN application through
Apache Twill, which caused a cleanup failure (and a confusing error message) upon
application termination.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4644">CDAP-4644</a> - Workflow logs now contain logs
from all of the actions executed by a workflow.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4842">CDAP-4842</a> - Added a <code class="docutils literal notranslate"><span class="pre">hydrator-test</span></code>
module that contains mock plugins for unit testing Hydrator plugins.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4925">CDAP-4925</a> - Added to the CDAP test
framework the ability to delete applications and artifacts, retrieve application
information, update an application, and write and remove properties for artifacts.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4955">CDAP-4955</a> - Added a ‘postaction’ Cask
Hydrator plugin type that runs at the end of a pipeline run, irregardless of whether the
run succeeded or failed.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-5001">CDAP-5001</a> - Downloading an explore query
from the CDAP UI will now stream the results directly to the client.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-5037">CDAP-5037</a> - Added a configuration property
to Cask Hydrator TimePartitionedFileSet (TPFS) sinks that will clean out data that is
older than a threshold amount of time.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-5039">CDAP-5039</a> - Added runtime macros to
database and post-action Cask Hydrator plugins.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-5042">CDAP-5042</a> - Added a <code class="docutils literal notranslate"><span class="pre">numSplits</span></code>
configuration property to Cask Hydrator database sources to allow users to configure how
many splits should be used for an import query.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-5046">CDAP-5046</a> - The CDAP UI now allows a
plugin developer to use a “textarea” in node configurations for displaying a plugin
property.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-5075">CDAP-5075</a> - Programs now have a
<code class="docutils literal notranslate"><span class="pre">logical.start.time</span></code> runtime argument that is populated by the system to be the start
time of the program. The argument can be overridden just as other runtime arguments.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-5082">CDAP-5082</a> - Added support for Spark
streaming (to interact with the transactional datasets in CDAP), and support for
concurrent Spark execution through Workflow forking.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-5178">CDAP-5178</a> - Changed the format of the Cask
Hydrator configuration. All pipeline stages are now together in a “stages” array instead
of being broken up into separate “source”, “transforms”, and “sinks” arrays.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-5181">CDAP-5181</a> - Added an HTTP RESTful endpoint
to retrieve the state of all nodes in a workflow.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-5182">CDAP-5182</a> - Added an API to retrieve the
properties that were used to configure (or reconfigure) a dataset.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-5207">CDAP-5207</a> - Removed dependency on Guava
from the <code class="docutils literal notranslate"><span class="pre">cdap-proto</span></code> module.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-5228">CDAP-5228</a> - Added support for CDH 5.7.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-5330">CDAP-5330</a> - The stream creation endpoint
now accepts a stream configuration (with TTL, description, format specification, and
notification threshold).</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-5376">CDAP-5376</a> - Added an API for MapReduce to
retrieve information about the enclosing workflow, including its run ID.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-5378">CDAP-5378</a> - Added access to workflow
information in a Spark program when it is executed inside a workflow.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-5424">CDAP-5424</a> - Added the ability to track the
lineage of external sources and sinks in a Cask Hydrator pipeline.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-5512">CDAP-5512</a> - Extended the workflow APIs to
allow the use of plugins.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-5664">CDAP-5664</a> - Introduced a <code class="docutils literal notranslate"><span class="pre">referenceName</span></code>
property (used for lineage and annotation metadata) into all external sources and sinks.
This needs to be set before using any of these plugins.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-5779">CDAP-5779</a> - Upgraded the Tephra version in
CDAP to 0.7.1.</li>
</ul>
</div>
<div class="section" id="id87">
<h3>Bug Fixes<a class="headerlink" href="#id87" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3498">CDAP-3498</a> - Upgraded CDAP to use
Apache Twill <code class="docutils literal notranslate"><span class="pre">0.7.0-incubating</span></code> with numerous new features, improvements, and bug
fixes. See the <a class="reference external" href="http://twill.apache.org/releases/">Apache Twill release notes</a> for details.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3584">CDAP-3584</a> - Upon transaction rollback, a
<code class="docutils literal notranslate"><span class="pre">PartitionedFileSet</span></code> now rolls back the files for the partitions that were added and/or
removed in that transaction.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3749">CDAP-3749</a> - Fixed a bug with the database
plugins that required a password to be specified if the user was specified, even if the
password was empty.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4060">CDAP-4060</a> - Added the status for custom
actions in workflow diagrams.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4143">CDAP-4143</a> - Fixed a problem with the
database source where a semicolon at the end of the query would cause an error.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4692">CDAP-4692</a> - The CDAP UI now prevents users
from accidentally losing their DAG by showing a browser-native popup for a confirmation
before navigating away from the Cask Hydrator Studio view.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4695">CDAP-4695</a> - Fixed an issue in the Windows
CDAP SDK where streams could not be deleted.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4735">CDAP-4735</a> - Fixed an issue that made Java
extensions unavailable to programs, fixing the JavaScript-based Hydrator transforms under Java 8.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4908">CDAP-4908</a> - Removed <code class="docutils literal notranslate"><span class="pre">tableName</span></code> as a
required setting from database sources, since the <code class="docutils literal notranslate"><span class="pre">importQuery</span></code> is sufficient.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4921">CDAP-4921</a> - Renamed the Hydrator
<code class="docutils literal notranslate"><span class="pre">Teradata</span></code> batch source to <code class="docutils literal notranslate"><span class="pre">Database</span></code>. The previous <code class="docutils literal notranslate"><span class="pre">Database</span></code> source is no longer
supported.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4982">CDAP-4982</a> - Changed the Cask Hydrator
LogParser transform <code class="docutils literal notranslate"><span class="pre">logFormat</span></code> field from a textbox to a dropdown.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-5041">CDAP-5041</a> - Changed several
<code class="docutils literal notranslate"><span class="pre">ExploreConnection</span></code> methods to be no-ops instead of throwing exceptions.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-5062">CDAP-5062</a> - Added a <code class="docutils literal notranslate"><span class="pre">fetch.size</span></code>
connection setting to the JDBC driver to control the number of rows fetched per database
cursor, and increased the default fetch size from 50 to 1000.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-5092">CDAP-5092</a> - Fixed a problem that prevented
applications written in Scala from being deployed.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-5103">CDAP-5103</a> - Fixed a problem so that when the
schema for a view was not explicitly specified, the view system metadata will include the
default schema for the specified format if that is available.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-5131">CDAP-5131</a> - Fixed a problem when filtering
plugins by their extension plugin type; filtering by the extensions plugin type was
returning extra results for any plugins that did not have an extension.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-5177">CDAP-5177</a> - Fixed a problem with
PartitionConsumer not appropriately handling partitions that had been deleted since they
were added to the working set.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-5241">CDAP-5241</a> - Fixed a problem with metadata
for a dataset not being deleted when a dataset was deleted.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-5267">CDAP-5267</a> - Fixed a problem with the
<code class="docutils literal notranslate"><span class="pre">PartitionFilter.ALWAYS_MATCH</span></code> not working as an input partition filter.
<code class="docutils literal notranslate"><span class="pre">PartitionFilter</span></code> is now serialized into one key of the runtime arguments, to support
serialization of <code class="docutils literal notranslate"><span class="pre">PartitionFilter.ALWAYS_MATCH</span></code>. If there are additional fields in the
<code class="docutils literal notranslate"><span class="pre">PartitionFilter</span></code> that do not exist in the partitioning, the filter will then never match.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-5272">CDAP-5272</a> - Fixed a problem with a null
pointer exception when null values were written to a database sink in Cask Hydrator.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-5280">CDAP-5280</a> - Corrected the documentation of
the Query HTTP RESTful API for the retrieving of the status of a query.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-5297">CDAP-5297</a> - Fixed a problem with the CDAP
UI not supporting pipelines created using previous versions of Cask Hydrator. The UI now
shows appropriate information to upgrade the pipeline to be able to view it in the UI.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-5417">CDAP-5417</a> - Fixed an issue with running
the CDAP examples in the CDAP SDK under Windows by setting appropriate memory requirements
in the <code class="docutils literal notranslate"><span class="pre">cdap.bat</span></code> start script.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-5460">CDAP-5460</a> - Fixed a problem with the
workflow Spark programs status not being updated in the CDAP UI on the program list
screen when it is run as a part of Workflow.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-5463">CDAP-5463</a> - Fixed an issue when changing
the number of instances of a worker or service.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-5513">CDAP-5513</a> - Fixed a problem with the
update of metadata indexes so that search results reflect metadata updates correctly.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-5550">CDAP-5550</a> - Fixed a problem with the
workflow statistics HTTP RESTful endpoint. The endpoint now has a default limit of 10 and
a default interval of 10 seconds.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-5557">CDAP-5557</a> - Fixed a problem of not showing
an appropriate error message in the node configuration when the CDAP backend returns 404
for a plugin property.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-5583">CDAP-5583</a> - Added the ability to support
multiple sources in the CDAP UI while constructing a pipeline.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-5619">CDAP-5619</a> - Fixed a problem with the
import of a pipeline configuration. If the imported pipeline config doesn’t have
artifact information for a plugin, the CDAP UI now defaults to the latest artifact from
the list of artifacts sent by the backend.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-5629">CDAP-5629</a> - Fixed a problem with losing
metadata after changing the stream format on a MapR cluster by avoiding the use of Hive
keywords in the CLF format field names; the ‘date’ field was renamed to ‘request_time’.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-5634">CDAP-5634</a> - Fixed a performance issue when
rendering/scrolling through large input or output schemas for a plugin in the CDAP UI.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-5652">CDAP-5652</a> - Added command line interface
command to retrieve the workflow node states.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-5793">CDAP-5793</a> - CDAP Explore jobs properly use
the latest/updated delegation tokens.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-5844">CDAP-5844</a> - Fixed a problem with the
updating of the HDFS delegation token for HA mode.</li>
</ul>
</div>
<div class="section" id="id88">
<h3>Deprecated and Removed Features<a class="headerlink" href="#id88" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li>See the <a class="reference external" href="http://docs.cask.co/cdap/3.4.0/en/reference-manual/javadocs/index.html#javadocs">CDAP 3.4.0 Javadocs</a>
for a list of deprecated and removed APIs.</li>
<li>As of <em>CDAP v3.4.0</em>, <em>Metadata Update Notifications</em> have been deprecated, pending
removal in a later version. The <a class="reference external" href="http://docs.cask.co/cdap/3.4.0/en/developers-manual/building-blocks/audit-logging.html#audit-logging">CDAP Audit Notifications</a>
contain
notifications for metadata changes. Please change all uses of <em>Metadata Update
Notifications</em> to consume only those messages from the audit feed that have the <code class="docutils literal notranslate"><span class="pre">type</span></code>
field set to <code class="docutils literal notranslate"><span class="pre">METADATA_CHANGE</span></code>.</li>
</ul>
</div>
<div class="section" id="known-issues-340">
<span id="id89"></span><h3>Known Issues<a class="headerlink" href="#known-issues-340" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li>After upgrading CDAP from a pre-3.0 version, any unprocessed metrics data in Kafka will
be lost and <em>WARN</em> log messages will be logged that tell about the inability to process
old data in the old format.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-797">CDAP-797</a> -
When running secure Hadoop clusters, debug logs from MapReduce programs are not
available.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-1007">CDAP-1007</a> -
If the Hive Metastore is restarted while the CDAP Explore Service is running, the
Explore Service remains alive, but becomes unusable. To correct, restart the CDAP Master—which will restart all services—as described under “Starting CDAP Services”
for your particular Hadoop distribution in the <a class="reference external" href="http://docs.cask.co/cdap/3.4.0/en/admin-manual/installation/index.html#installation-index">Installation documentation</a>.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-1587">CDAP-1587</a> -
CDAP internally creates tables in the “user” space that begin with the word
<code class="docutils literal notranslate"><span class="pre">&quot;system&quot;</span></code>. User datasets with names starting with <code class="docutils literal notranslate"><span class="pre">&quot;system&quot;</span></code> can conflict if they
were to match one of those names. To avoid this, do not start any datasets with the word
<code class="docutils literal notranslate"><span class="pre">&quot;system&quot;</span></code>.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2632">CDAP-2632</a> -
The application in the <a class="reference external" href="https://github.com/cdap-guides/cdap-kafka-ingest-guide/tree/release/cdap-3.0-compatible">cdap-kafka-ingest-guide</a>
does not run on Ubuntu 14.x as of CDAP 3.0.x.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2721">CDAP-2721</a> -
Metrics for <a class="reference external" href="http://docs.cask.co/cdap/3.4.0/en/developers-manual/building-blocks/datasets/fileset.html#datasets-fileset">FileSets</a>
can show zero values even if there is
data present, because FileSets do not emit metrics (<a class="reference external" href="https://issues.cask.co/browse/CDAP-587">CDAP-587</a>).</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2831">CDAP-2831</a> -
A workflow that is scheduled by time will not be run between the failure of the primary
master and the time that the secondary takes over. This scheduled run will not be
triggered at all.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2920">CDAP-2920</a> - Spark jobs on a
Kerberos-enabled CDAP cluster cannot run longer than the delegation token expiration.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2945">CDAP-2945</a> -
If the input partition filter for a PartitionedFileSet does not match any partitions,
MapReduce jobs can fail.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3000">CDAP-3000</a> -
The Workflow token is in an inconsistent state for nodes in a fork while the nodes of
the fork are still running. It becomes consistent after the join.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3221">CDAP-3221</a> -
When running in Standalone CDAP, if a MapReduce job fails repeatedly, then the SDK
hits an out-of-memory exception due to <code class="docutils literal notranslate"><span class="pre">perm</span> <span class="pre">gen</span></code>. The Standalone needs restarting at
this point.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3262">CDAP-3262</a> -
For Microsoft Windows, the Standalone CDAP scripts can fail when used with a JAVA_HOME
that is defined as a path with spaces in it. A workaround is to use a definition of
JAVA_HOME that does not include spaces, such as <code class="docutils literal notranslate"><span class="pre">C:\PROGRA~1\Java\jdk1.7.0_79\bin</span></code> or
<code class="docutils literal notranslate"><span class="pre">C:\ProgramData\Oracle\Java\javapath</span></code>.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3492">CDAP-3492</a> -
In the CDAP CLI, executing <code class="docutils literal notranslate"><span class="pre">select</span> <span class="pre">*</span></code> from a dataset with many fields generates an error.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3641">CDAP-3641</a> -
A RESTful API call to retrieve workflow statistics hangs if units (such as “s” for
seconds) are not provided as part of the query.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3750">CDAP-3750</a> -
If a table schema contains a field name that is a reserved word in the Hive DDL, <code class="docutils literal notranslate"><span class="pre">'enable</span> <span class="pre">explore'</span></code> fails.</li>
</ul>
</div>
</div>
<div class="section" id="release-3-3-3">
<h2><a class="reference external" href="http://docs.cask.co/cdap/3.3.3/index.html">Release 3.3.3</a><a class="headerlink" href="#release-3-3-3" title="Permalink to this headline">🔗</a></h2>
<div class="section" id="bug-fix">
<h3>Bug Fix<a class="headerlink" href="#bug-fix" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-5350">CDAP-5350</a> - Fixed an issue that prevented
MapReduce programs from running on clusters with encryption.</li>
</ul>
</div>
</div>
<div class="section" id="release-3-3-2">
<h2><a class="reference external" href="http://docs.cask.co/cdap/3.3.2/index.html">Release 3.3.2</a><a class="headerlink" href="#release-3-3-2" title="Permalink to this headline">🔗</a></h2>
<div class="section" id="id90">
<h3>Improvements<a class="headerlink" href="#id90" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-5047">CDAP-5047</a> - Added a <a class="reference external" href="http://docs.cask.co/cdap/3.3.2/en/cdap-apps/hydrator/hydrator-plugins/batchsources/azureblobstore.html">Batch Source Plugin</a>
to read from Microsoft Azure Blob Storage.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-5134">CDAP-5134</a> - Added support for CDH 5.6 to CDAP.</li>
</ul>
</div>
<div class="section" id="id91">
<h3>Bug Fixes<a class="headerlink" href="#id91" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4967">CDAP-4967</a> - Fixed a schema-parsing bug
that prevented the use of schemas where a record is used both as a top-level field and
also used inside a different record field.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-5019">CDAP-5019</a> - Worked around two issues
(<a class="reference external" href="https://issues.apache.org/jira/browse/SPARK-13441">SPARK-13441</a>
and <a class="reference external" href="https://issues.apache.org/jira/browse/YARN-4727">YARN-4727</a>) that prevented
launching Spark jobs on CDH (Cloudera Data Hub) clusters managed with Cloudera Manager
when using Spark 1.4 or greater.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-5063">CDAP-5063</a> - Fixed a problem with
the CDAP Master not starting when CDAP and the HiveServer2 services are running on the
same node in an Ambari cluster.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-5076">CDAP-5076</a> - Fixed a problem with the CDAP
CLI command “update app” that was parsing the application config incorrectly.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-5094">CDAP-5094</a> - Fixed a problem where the explore
schema fileset property was being ignored unless an explore format was also present.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-5137">CDAP-5137</a> - Fix a problem with Spark jobs
not being submitted to the appropriate YARN scheduler queue set for the namespace.</li>
</ul>
</div>
</div>
<div class="section" id="release-3-3-1">
<h2><a class="reference external" href="http://docs.cask.co/cdap/3.3.1/index.html">Release 3.3.1</a><a class="headerlink" href="#release-3-3-1" title="Permalink to this headline">🔗</a></h2>
<div class="section" id="id92">
<h3>Improvements<a class="headerlink" href="#id92" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4602">CDAP-4602</a> - Updated CDAP to use
Tephra 0.6.5.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4708">CDAP-4708</a> - Added system metadata to
existing entities.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4723">CDAP-4723</a> - Improved the Hydrator plugin
archetypes to include build steps to build the deployment JSON for the artifact.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4773">CDAP-4773</a> - Improved the error logging for
the Master Stream service when it can’t connect to the CDAP AppFabric server.</li>
</ul>
</div>
<div class="section" id="id93">
<h3>Bug Fixes<a class="headerlink" href="#id93" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4117">CDAP-4117</a> - Fixed an issue of not using
the correct user to access HDFS when submitting a YARN application through Apache Twill,
which caused cleanup failure on application termination.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4613">CDAP-4613</a> - Fixed a problem with tooltips
not appearing in Flow and Workflow diagrams displayed in the Firefox browser.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4679">CDAP-4679</a> - The Hydrator UI now prevents
drafts from being created with a name of an already-existing draft. This prevents
overwriting of existing drafts.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4688">CDAP-4688</a> - Improved the metadata search
to return matching entities from both the specified namespace and the system namespace.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4689">CDAP-4689</a> - Fixed a problem when using an
Hbase sink as one of multiple sinks in a Hydrator pipeline.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4720">CDAP-4720</a> - Fixed an issue where system
metadata updates were not being published to Kafka.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4721">CDAP-4721</a> - Fixed an issue where metadata
updates wouldn’t be sent when certain entities were deleted.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4740">CDAP-4740</a> - Added validation to the JSON
imported in the Hydrator UI.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4741">CDAP-4741</a> - Fixed a bug with deleting
artifact metadata when an artifact was deleted.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4743">CDAP-4743</a> - Fixed the Node.js server proxy
to handle all backend errors (with and without statusCodes).</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4745">CDAP-4745</a> - Fixed a bug in the Hydrator
upgrade tool which caused drafts to not get upgraded.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4753">CDAP-4753</a> - Fixed the Hydrator Stream
source to not assume an output schema. This is valid when a pipeline is created outside
Hydrator UI.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4754">CDAP-4754</a> - Fixed ObjectStore to work when
parameterized with custom classes.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4767">CDAP-4767</a> - Fixed an issue where delegation token
cancellation of CDAP program was affecting CDAP master services.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4770">CDAP-4770</a> - Fixed the Cask Hydrator UI to
automatically reconnect with the CDAP backend when the backend restarts.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4771">CDAP-4771</a> - Fixed an issue in Cloudera
Manager installations where CDAP container logs would go to the stdout file instead of the
master log.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4784">CDAP-4784</a> - Fixed an issue where the
IndexedTable was dropping indices upon row updates.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4785">CDAP-4785</a> - Fixed a problem in the upgrade
tool where deleted datasets would cause it to throw a NullPointerException.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4790">CDAP-4790</a> - Fixed an issue where the Hbase
implementation of the Table API returned all rows, when the correct response should have
been an empty set of columns.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4800">CDAP-4800</a> - Fixed a problem with the error
message returned when loading an artifact with an invalid range.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4806">CDAP-4806</a> - Fixed the PartitionedFileSet’s
DynamicPartitioner to work with Avro OutputFormats.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4829">CDAP-4829</a> - Fixed a Validator Transform
function generator in the Hydrator UI.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4831">CDAP-4831</a> - Allows user-scoped plugins to
surface the correct widget JSON in the Hydrator UI.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4832">CDAP-4832</a> - Added the ErrorDataset as an
option on widget JSON in Hydrator plugins.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4836">CDAP-4836</a> - Fixed a spacing issue for
metrics showing in Pipeline diagrams of the Hydrator UI.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4853">CDAP-4853</a> - Fixed issues with the Hydrator
UI widgets for the Hydrator Kafka real-time source, JMS real-time source, and CloneRecord
transform.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4865">CDAP-4865</a> - Enhanced the CDAP SDK to be
able to publish metadata updates to an external Kafka, identified by the configuration
property <code class="docutils literal notranslate"><span class="pre">metadata.updates.kafka.broker.list</span></code>. Publishing can be enabled by setting
<code class="docutils literal notranslate"><span class="pre">metadata.updates.publish.enabled</span></code> to true. Updates are published to the Kafka topic
identified by the property <code class="docutils literal notranslate"><span class="pre">metadata.updates.kafka.topic</span></code>.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4877">CDAP-4877</a> - Fixed errors in Cask Hydrator
Plugins. Two plugin documents (<code class="docutils literal notranslate"><span class="pre">core-plugins/docs/Database-batchsink.md</span></code> and
<code class="docutils literal notranslate"><span class="pre">core-plugins/docs/Database-batchsource.md</span></code>) were removed, as the plugins have been moved
from <em>core-plugins</em> to <em>database-plugins</em> (to <code class="docutils literal notranslate"><span class="pre">database-plugins/docs/Database-batchsink.md</span></code>
and <code class="docutils literal notranslate"><span class="pre">database-plugins/docs/Database-batchsource.md</span></code>).</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4889">CDAP-4889</a> - Fixed an issue with upgrading
HBase tables while using the CDAP Upgrade Tool.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4894">CDAP-4894</a> - Fixed an issue with CDAP
coprocessors that caused HBase tables to be disabled after upgrading the cluster to a
highly-available file system.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4906">CDAP-4906</a> - Fixed the CDAP Upgrade Tool to
return a non-zero exit status upon error during upgrade.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4924">CDAP-4924</a> - Fixed a PermGen memory leak
that occurred while deploying multiple applications with database plugins.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4927">CDAP-4927</a> - Fixed the CDAP Explore
Service JDBC driver to do nothing instead of throwing an exception when a commit is
called.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4950">CDAP-4950</a> - Added an <code class="docutils literal notranslate"><span class="pre">'enableAutoCommit'</span></code>
property to the Cask Hydrator database plugins to enable the use of JDBC drivers that,
similar to the Hive JDBC driver, do not allow commits.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4951">CDAP-4951</a> - Changed the upload timeout from the
CDAP CLI from 15 seconds to unlimited.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4975">CDAP-4975</a> - Pass ResourceManager delegation tokens
in the proper format in secure Hadoop HA clusters.</li>
</ul>
</div>
<div class="section" id="id94">
<h3>Deprecated and Removed Features<a class="headerlink" href="#id94" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li>See the <a class="reference external" href="http://docs.cask.co/cdap/3.3.1/en/reference-manual/javadocs/index.html#javadocs">CDAP 3.3.1 Javadocs</a>
for a list of deprecated and removed APIs.</li>
<li>The properties <code class="docutils literal notranslate"><span class="pre">router.ssl.webapp.bind.port</span></code>, <code class="docutils literal notranslate"><span class="pre">router.webapp.bind.port</span></code>,
<code class="docutils literal notranslate"><span class="pre">router.webapp.enabled</span></code> have been deprecated and will be removed in a future version.</li>
</ul>
</div>
<div class="section" id="id95">
<h3>Known Issues<a class="headerlink" href="#id95" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li>After upgrading CDAP from a pre-3.0 version, any unprocessed metrics data in Kafka will
be lost and <em>WARN</em> log messages will be logged that tell about the inability to process
old data in the old format.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-797">CDAP-797</a> -
When running secure Hadoop clusters, debug logs from MapReduce programs are not
available.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-1007">CDAP-1007</a> -
If the Hive Metastore is restarted while the CDAP Explore Service is running, the
Explore Service remains alive, but becomes unusable. To correct, restart the CDAP Master—which will restart all services—as described under “Starting CDAP Services”
for your particular Hadoop distribution in the <a class="reference external" href="http://docs.cask.co/cdap/3.3.1/en/admin-manual/installation/index.html#installation-index">Installation documentation</a>.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-1587">CDAP-1587</a> -
CDAP internally creates tables in the “user” space that begin with the word
<code class="docutils literal notranslate"><span class="pre">&quot;system&quot;</span></code>. User datasets with names starting with <code class="docutils literal notranslate"><span class="pre">&quot;system&quot;</span></code> can conflict if they
were to match one of those names. To avoid this, do not start any datasets with the word
<code class="docutils literal notranslate"><span class="pre">&quot;system&quot;</span></code>.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2632">CDAP-2632</a> -
The application in the <a class="reference external" href="https://github.com/cdap-guides/cdap-kafka-ingest-guide/tree/release/cdap-3.0-compatible">cdap-kafka-ingest-guide</a>
does not run on Ubuntu 14.x as of CDAP 3.0.x.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2721">CDAP-2721</a> -
Metrics for <a class="reference external" href="http://docs.cask.co/cdap/3.3.1/en/developers-manual/building-blocks/datasets/fileset.html#datasets-fileset">FileSets</a>
can show zero values even if there is
data present, because FileSets do not emit metrics (<cite>CDAP-587
&lt;https://issues.cask.co/browse/CDAP-587&gt;</cite>).</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2831">CDAP-2831</a> -
A workflow that is scheduled by time will not be run between the failure of the primary
master and the time that the secondary takes over. This scheduled run will not be
triggered at all.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2945">CDAP-2945</a> -
If the input partition filter for a PartitionedFileSet does not match any partitions,
MapReduce jobs can fail.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3000">CDAP-3000</a> -
The Workflow token is in an inconsistent state for nodes in a fork while the nodes of
the fork are still running. It becomes consistent after the join.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3221">CDAP-3221</a> -
When running in Standalone CDAP, mode, if a MapReduce job fails repeatedly, then the SDK
hits an out-of-memory exception due to <code class="docutils literal notranslate"><span class="pre">perm</span> <span class="pre">gen</span></code>. The Standalone needs restarting at
this point.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3262">CDAP-3262</a> -
For Microsoft Windows, the Standalone CDAP scripts can fail when used with a JAVA_HOME
that is defined as a path with spaces in it. A workaround is to use a definition of
JAVA_HOME that does not include spaces, such as <code class="docutils literal notranslate"><span class="pre">C:\PROGRA~1\Java\jdk1.7.0_79\bin</span></code> or
<code class="docutils literal notranslate"><span class="pre">C:\ProgramData\Oracle\Java\javapath</span></code>.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3492">CDAP-3492</a> -
In the CDAP CLI, executing <code class="docutils literal notranslate"><span class="pre">select</span> <span class="pre">*</span></code> from a dataset with many fields generates an error.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3641">CDAP-3641</a> -
A RESTful API call to retrieve workflow statistics hangs if units (such as “s” for
seconds) are not provided as part of the query.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3750">CDAP-3750</a> -
If a table schema contains a field name that is a reserved word in the Hive DDL, <code class="docutils literal notranslate"><span class="pre">'enable</span> <span class="pre">explore'</span></code> fails.</li>
</ul>
</div>
</div>
<div class="section" id="release-3-3-0">
<h2><a class="reference external" href="http://docs.cask.co/cdap/3.3.0/index.html">Release 3.3.0</a><a class="headerlink" href="#release-3-3-0" title="Permalink to this headline">🔗</a></h2>
<div class="section" id="id96">
<h3>New Features<a class="headerlink" href="#id96" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-961">CDAP-961</a> -
Added on demand (dynamic) dataset instantiation through program runtime context.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2303">CDAP-2303</a> -
Added lookup capability in context that can be used in existing Script, ScriptFilter and Validator transforms.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3514">CDAP-3514</a> -
Added an endpoint to get a count of active queries: <code class="docutils literal notranslate"><span class="pre">/v3/namespaces/&lt;namespace-id&gt;/data/explore/queries/count</span></code>.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3857">CDAP-3857</a> -
Added experimental support for running ETL Batch applications on Spark. Introduced an ‘engine’ setting in the
configuration that defaults to <code class="docutils literal notranslate"><span class="pre">'mapreduce'</span></code>, but can be set to <code class="docutils literal notranslate"><span class="pre">'spark'</span></code>.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3944">CDAP-3944</a> -
Added support to PartitionConsumer for concurrency, plus a limit and filter on read.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3945">CDAP-3945</a> -
Added support for limiting the number of concurrent schedule runs.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4016">CDAP-4016</a> -
Added Java-8 support for Script transforms.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4022">CDAP-4022</a> -
Added RESTful APIs to start or stop multiple programs.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4023">CDAP-4023</a> -
Added CLI commands to stop, start, restart, or get status of programs in an application.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4043">CDAP-4043</a> -
Added support for ETL transforms written in Python.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4128">CDAP-4128</a> -
Added a new JavaScript transform that can emit records using an emitter.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4135">CDAP-4135</a> -
Added the capability for MapReduce and Spark programs to localize additional resources during setup.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4228">CDAP-4228</a> -
Added the ability to configure which artifact a Hydrator plugin should use.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4230">CDAP-4230</a> -
Added DAGs to ETL pipelines, which will allow users to fork and merge. ETLConfig has been
updated to allow representing a DAG.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4235">CDAP-4235</a> -
Added AuthorizationPlugin, for pluggable authorization.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4263">CDAP-4263</a> -
Added metadata support for stream views.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4270">CDAP-4270</a> -
Added CLI support for metadata and lineage.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4280">CDAP-4280</a> -
Added the ability to add metadata to artifacts.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4289">CDAP-4289</a> -
Added RESTful APIs to set and get properties for an artifact.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4264">CDAP-4264</a> -
Added support for automatically annotating CDAP entities with system metadata when they are created or updated.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4285">CDAP-4285</a> -
Added an authorization plugin that uses a system dataset to manage ACLs.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4403">CDAP-4403</a> -
Moved Hydrator plugins from the CDAP repository as cdap-etl-lib into its own repository.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4591">CDAP-4591</a> -
Improved Metadata Indexing and Search to support searches on words in value and tags.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4592">CDAP-4592</a> -
Schema fields are stored as Metadata and are searchable.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4658">CDAP-4658</a> -
Added capability in CDAP UI to display system tags.</li>
</ul>
</div>
<div class="section" id="id97">
<h3>Improvements<a class="headerlink" href="#id97" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3079">CDAP-3079</a> -
Table datasets, and any other dataset that implements <code class="docutils literal notranslate"><span class="pre">RecordWritable&lt;StructuredRecord&gt;</span></code>,
can now be written to using Hive.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3887">CDAP-3887</a> -
The CDAP Router now has a configurable timeout for idle connections, with a default
timeout of 15 seconds.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4045">CDAP-4045</a> -
A new property master.collect.containers.log has been added to cdap-site.xml, which
determines if container logs are streamed back to the cdap-master process log. (This has
always been the default behavior). For MapR installations, this must be turned off (set
to false).</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4133">CDAP-4133</a> -
Added ability to retrieve the live-info for the AppFabric system service.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4209">CDAP-4209</a> -
Added a method to <code class="docutils literal notranslate"><span class="pre">ObjectMappedTable</span></code> and <code class="docutils literal notranslate"><span class="pre">ObjectStore</span></code> to retrieve a specific
number of splits between a start and end keys.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4233">CDAP-4233</a> -
Messages logged by Hydrator are now prefixed with the name of the stage that logged them.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4301">CDAP-4301</a> -
Added support for CDH5.5</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4392">CDAP-4392</a> -
Upgraded netty-http dependency in CDAP to 0.14.0.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4444">CDAP-4444</a> -
Make <code class="docutils literal notranslate"><span class="pre">xmllint</span></code> dependency optional and allow setting variables to skip configuration
file parsing.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4453">CDAP-4453</a> -
Added a schema validation—for sources, transforms, and sinks—that will
validate the pipeline stages schema during deployment, and report any issues.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4518">CDAP-4518</a> -
CDAP Master service will now log important configuration settings on startup.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4523">CDAP-4523</a> -
Added the config setting <code class="docutils literal notranslate"><span class="pre">master.startup.checks.enabled</span></code> to control whether CDAP
Master startup checks are run or not.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4536">CDAP-4536</a> -
Improved the installation experience by adding to the CDAP Master service checks of
pre-requisites such as file system permissions, availability of components such as YARN
and HBase, resource availability during startup, and to error out if any of the
pre-requisites fail.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4548">CDAP-4548</a> -
Added a config setting ‘master.collect.app.containers.log’ that can be set to ‘false’ to
disable streaming of application logs back to the CDAP Master log.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4598">CDAP-4598</a> -
Added an error message when a required field is not provided when configuring Hydrator
pipeline.</li>
</ul>
</div>
<div class="section" id="id98">
<h3>Bug Fixes<a class="headerlink" href="#id98" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-1174">CDAP-1174</a> -
Prefix start script functions with <code class="docutils literal notranslate"><span class="pre">'cdap'</span></code> to prevent namespace collisions.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2470">CDAP-2470</a> -
Added a check to cause a DB (source or sink) pipeline to fail during deployment if the
table (source or sink) was not found, or if an incorrect connection string was provided.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3345">CDAP-3345</a> -
Fixed a bug where the TTL for datasets was incorrect; it was reduced by (a factor of
1000) after an upgrade. After running the upgrade tool, please make sure the TTL values
of tables are as expected.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3542">CDAP-3542</a> -
Fixed an issue where the failure of a program running in a workflow fork node was
causing other programs in the same fork node to remain in the RUNNING state, even after
the Workflow was completed.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3694">CDAP-3694</a> -
Fixed test failures in the PurchaseHistory, StreamConversion, and WikipediaPipeline
example apps included in the CDAP SDK.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3742">CDAP-3742</a> -
Fixed a bug where certain MapReduce metrics were not being properly emitted when using
multiple outputs.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3761">CDAP-3761</a> -
Fixed a problem with DBSink column names not being used to filter input record fields
before writing to a DBSink.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3807">CDAP-3807</a> -
Added a fix for case sensitivity handling in DBSink.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3815">CDAP-3815</a> -
Fixed an issue where the regex filter for S3 Batch Source wasn’t getting applied correctly.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3861">CDAP-3861</a> -
Fixed an issue about stopping all dependent services when a service is stopped.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3900">CDAP-3900</a> -
Fixed a bug when querying for logs of deleted program runs.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3902">CDAP-3902</a> -
Fixed a problem with dataset performance degradation because of making multiple remote
calls for each “get dataset” request.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3924">CDAP-3924</a> -
Fixed QueryClient to work against HTTPS.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4000">CDAP-4000</a> -
Fixed an issue where a stream that has a view could not be deleted cleanly.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4067">CDAP-4067</a> -
Fixed an issue where socket connections to the TransactionManager were not being closed.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4092">CDAP-4092</a> -
Fixes an issue that causes worker threads to go into an infinite recursion while
exceptions are being thrown in channel handlers.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4112">CDAP-4112</a> -
Fixed a bug that prevented applications from using HBase directly.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4119">CDAP-4119</a> -
Fixed a problem where when CDAP Master switched from active to standby, the programs
that were running were marked as failed.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4240">CDAP-4240</a> -
Fixed a problem in the CLI command used to load an artifact, where the wrong artifact name
and version was used if the artifact name ends with a number.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4294">CDAP-4294</a> -
Fixed a problem where plugins from another namespace were visible when creating an
application using a system artifact.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4316">CDAP-4316</a> -
Fixed a problem with the CLI attempting to connect to CDAP when the hostname and port
were incorrect.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4366">CDAP-4366</a> -
Improved error message when stream views were not found.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4393">CDAP-4393</a> -
Fixed an issue where tags search were failing for certain tags.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4141">CDAP-4141</a> -
Fixed node.js version checking for the <code class="docutils literal notranslate"><span class="pre">cdap</span> <span class="pre">sdk</span></code> script in the CDAP SDK.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4373">CDAP-4373</a> -
Fixed a problem that prevented MapReduce jobs from being run when the Resource Manager
switches from active to standby in a Kerberos-enabled HA cluster.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4384">CDAP-4384</a> -
Fixed an issue that prevents streams from being read in HA HDFS mode.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4526">CDAP-4526</a> -
Fixed init scripts to print service status when stopped.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4534">CDAP-4534</a> -
Added configuration ‘router.bypass.auth.regex’ to exempt certain URLs from authentication.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4539">CDAP-4539</a> -
Fixed a problem in the init scripts that forced <code class="docutils literal notranslate"><span class="pre">cdap-kafka-server</span></code>, <code class="docutils literal notranslate"><span class="pre">cdap-router</span></code>,
and <code class="docutils literal notranslate"><span class="pre">cdap-auth-server</span></code> to have the Hive client installed.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4678">CDAP-4678</a> -
Fixed an issue where the logs and history list on a Hydrator pipeline view was not
updating on new runs.</li>
</ul>
</div>
<div class="section" id="id99">
<h3>Deprecated and Removed Features<a class="headerlink" href="#id99" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li>See the <a class="reference external" href="http://docs.cask.co/cdap/3.3.0/en/reference-manual/javadocs/index.html#javadocs">CDAP 3.3.0 Javadocs</a>
for a list of deprecated and removed APIs.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2481">CDAP-2481</a> -
Removed a deprecated endpoint to retrieve the status of a currently running node in a workflow.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2943">CDAP-2943</a> -
Removed the deprecated builder-style Flow API.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4128">CDAP-4128</a> -
Deprecated the Script transform.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4217">CDAP-4217</a> -
Deprecated createDataSchedule and createTimeSchedule methods in Schedules class and removed
deprecated Schedule constructor.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-4251">CDAP-4251</a> -
Removed deprecated fluent style API for Flow configuration. The only supported API is now the configurer style.</li>
</ul>
</div>
<div class="section" id="id100">
<h3>Known Issues<a class="headerlink" href="#id100" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li>After upgrading CDAP from a pre-3.0 version, any unprocessed metrics data in Kafka will
be lost and <em>WARN</em> log messages will be logged that tell about the inability to process
old data in the old format.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-797">CDAP-797</a> -
When running secure Hadoop clusters, debug logs from MapReduce programs are not
available.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-1007">CDAP-1007</a> -
If the Hive Metastore is restarted while the CDAP Explore Service is running, the
Explore Service remains alive, but becomes unusable. To correct, restart the CDAP Master—which will restart all services—as described under “Starting CDAP Services”
for your particular Hadoop distribution in the <a class="reference external" href="http://docs.cask.co/cdap/3.3.0/en/admin-manual/installation/index.html#installation-index">Installation documentation</a>.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-1587">CDAP-1587</a> -
CDAP internally creates tables in the “user” space that begin with the word
<code class="docutils literal notranslate"><span class="pre">&quot;system&quot;</span></code>. User datasets with names starting with <code class="docutils literal notranslate"><span class="pre">&quot;system&quot;</span></code> can conflict if they
were to match one of those names. To avoid this, do not start any datasets with the word
<code class="docutils literal notranslate"><span class="pre">&quot;system&quot;</span></code>.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2632">CDAP-2632</a> -
The application in the <a class="reference external" href="https://github.com/cdap-guides/cdap-kafka-ingest-guide/tree/release/cdap-3.0-compatible">cdap-kafka-ingest-guide</a>
does not run on Ubuntu 14.x as of CDAP 3.0.x.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2721">CDAP-2721</a> -
Metrics for <a class="reference external" href="http://docs.cask.co/cdap/3.3.0/en/developers-manual/building-blocks/datasets/fileset.html#datasets-fileset">FileSets</a>
can show zero values even if there is
data present, because FileSets do not emit metrics (<cite>CDAP-587
&lt;https://issues.cask.co/browse/CDAP-587&gt;</cite>).</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2831">CDAP-2831</a> -
A workflow that is scheduled by time will not be run between the failure of the primary
master and the time that the secondary takes over. This scheduled run will not be
triggered at all.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2945">CDAP-2945</a> -
If the input partition filter for a PartitionedFileSet does not match any partitions,
MapReduce jobs can fail.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3000">CDAP-3000</a> -
The Workflow token is in an inconsistent state for nodes in a fork while the nodes of
the fork are still running. It becomes consistent after the join.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3221">CDAP-3221</a> -
When running in Standalone CDAP, if a MapReduce job fails repeatedly, then the SDK
hits an out-of-memory exception due to <code class="docutils literal notranslate"><span class="pre">perm</span> <span class="pre">gen</span></code>. The Standalone needs restarting at
this point.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3262">CDAP-3262</a> -
For Microsoft Windows, the Standalone CDAP scripts can fail when used with a JAVA_HOME
that is defined as a path with spaces in it. A workaround is to use a definition of
JAVA_HOME that does not include spaces, such as <code class="docutils literal notranslate"><span class="pre">C:\PROGRA~1\Java\jdk1.7.0_79\bin</span></code> or
<code class="docutils literal notranslate"><span class="pre">C:\ProgramData\Oracle\Java\javapath</span></code>.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3492">CDAP-3492</a> -
In the CDAP CLI, executing <code class="docutils literal notranslate"><span class="pre">select</span> <span class="pre">*</span></code> from a dataset with many fields generates an error.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3641">CDAP-3641</a> -
A RESTful API call to retrieve workflow statistics hangs if units (such as “s” for
seconds) are not provided as part of the query.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3750">CDAP-3750</a> -
If a table schema contains a field name that is a reserved word in the Hive DDL, <code class="docutils literal notranslate"><span class="pre">'enable</span> <span class="pre">explore'</span></code> fails.</li>
</ul>
</div>
</div>
<div class="section" id="release-3-2-1">
<h2><a class="reference external" href="http://docs.cask.co/cdap/3.2.1/index.html">Release 3.2.1</a><a class="headerlink" href="#release-3-2-1" title="Permalink to this headline">🔗</a></h2>
<div class="section" id="id101">
<h3>New Features<a class="headerlink" href="#id101" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3951">CDAP-3951</a> -
Added the ability for S3 batch sources and sinks to set additional file system properties.</li>
</ul>
</div>
<div class="section" id="id102">
<h3>Improvements<a class="headerlink" href="#id102" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3870">CDAP-3870</a> -
Added logging and metrics support for <em>Script</em>, <em>ScriptFilter</em>, and <em>Validator</em> transforms.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3939">CDAP-3939</a> -
Improved artifact and application deployment failure handling.</li>
</ul>
</div>
<div class="section" id="id103">
<h3>Bug Fixes<a class="headerlink" href="#id103" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3342">CDAP-3342</a> -
Fixed a problem with the CDAP SDK unable to start on certain Windows machines by updating
the Hadoop native library in CDAP with a version that does not have a dependency on a
debug version of the Microsoft <code class="docutils literal notranslate"><span class="pre">msvcr100.dll</span></code>.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3815">CDAP-3815</a> -
Fixed an issue where the regex filter for S3 batch sources wasn’t being applied correctly.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3829">CDAP-3829</a> -
Fixed snapshot sinks so that the data is explorable as a <code class="docutils literal notranslate"><span class="pre">PartitionedFileSet</span></code>.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3833">CDAP-3833</a> -
Fixed snapshot sinks so that they can be read safely.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3859">CDAP-3859</a> -
Fixed a compilation error in the Maven application archetype.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3860">CDAP-3860</a> -
Fixed a bug where plugins, packaged in the same artifact as an application class, could not be used by that application class.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3891">CDAP-3891</a> -
Updated the documentation to remove references to application templates and adaptors that were removed as of CDAP 3.2.0.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3949">CDAP-3949</a> -
Fixed a problem with running certain examples on Linux systems by increasing the maximum
Java heap size of the Standalone SDK on Linux systems to 2048m.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3961">CDAP-3961</a> -
Fixed a missing dependency on <code class="docutils literal notranslate"><span class="pre">cdap-hbase-compat-1.1</span></code> package in the CDAP Master package.</li>
</ul>
</div>
<div class="section" id="id104">
<h3>Known Issues<a class="headerlink" href="#id104" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li>After upgrading CDAP from a pre-3.0 version, any unprocessed metrics data in Kafka will
be lost and <em>WARN</em> log messages will be logged that tell about the inability to process
old data in the old format.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-797">CDAP-797</a> -
When running secure Hadoop clusters, debug logs from MapReduce programs are not
available.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-1007">CDAP-1007</a> -
If the Hive Metastore is restarted while the CDAP Explore Service is running, the
Explore Service remains alive, but becomes unusable. To correct, restart the CDAP Master—which will restart all services—as described under “Starting CDAP Services”
for your particular Hadoop distribution in the <a class="reference external" href="http://docs.cask.co/cdap/3.2.1/en/admin-manual/installation/index.html#installation-index">Installation documentation</a>.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-1587">CDAP-1587</a> -
CDAP internally creates tables in the “user” space that begin with the word
<code class="docutils literal notranslate"><span class="pre">&quot;system&quot;</span></code>. User datasets with names starting with <code class="docutils literal notranslate"><span class="pre">&quot;system&quot;</span></code> can conflict if they
were to match one of those names. To avoid this, do not start any datasets with the word
<code class="docutils literal notranslate"><span class="pre">&quot;system&quot;</span></code>.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2632">CDAP-2632</a> -
The application in the <a class="reference external" href="https://github.com/cdap-guides/cdap-kafka-ingest-guide/tree/release/cdap-3.0-compatible">cdap-kafka-ingest-guide</a>
does not run on Ubuntu 14.x as of CDAP 3.0.x.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2721">CDAP-2721</a> -
Metrics for <a class="reference external" href="http://docs.cask.co/cdap/3.2.1/en/developers-manual/building-blocks/datasets/fileset.html#datasets-fileset">FileSets</a>
can show zero values even if there is
data present, because FileSets do not emit metrics (<cite>CDAP-587
&lt;https://issues.cask.co/browse/CDAP-587&gt;</cite>).</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2831">CDAP-2831</a> -
A workflow that is scheduled by time will not be run between the failure of the primary
master and the time that the secondary takes over. This scheduled run will not be
triggered at all.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2945">CDAP-2945</a> -
If the input partition filter for a PartitionedFileSet does not match any partitions,
MapReduce jobs can fail.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3000">CDAP-3000</a> -
The Workflow token is in an inconsistent state for nodes in a fork while the nodes of
the fork are still running. It becomes consistent after the join.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3221">CDAP-3221</a> -
When running in Standalone CDAP, if a MapReduce job fails repeatedly, then the SDK
hits an out-of-memory exception due to <code class="docutils literal notranslate"><span class="pre">perm</span> <span class="pre">gen</span></code>. The Standalone needs restarting at
this point.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3262">CDAP-3262</a> -
For Microsoft Windows, the Standalone CDAP scripts can fail when used with a JAVA_HOME
that is defined as a path with spaces in it. A workaround is to use a definition of
JAVA_HOME that does not include spaces, such as <code class="docutils literal notranslate"><span class="pre">C:\PROGRA~1\Java\jdk1.7.0_79\bin</span></code> or
<code class="docutils literal notranslate"><span class="pre">C:\ProgramData\Oracle\Java\javapath</span></code>.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3492">CDAP-3492</a> -
In the CDAP CLI, executing <code class="docutils literal notranslate"><span class="pre">select</span> <span class="pre">*</span></code> from a dataset with many fields generates an error.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3641">CDAP-3641</a> -
A RESTful API call to retrieve workflow statistics hangs if units (such as “s” for
seconds) are not provided as part of the query.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3750">CDAP-3750</a> -
If a table schema contains a field name that is a reserved word in the Hive DDL, <code class="docutils literal notranslate"><span class="pre">'enable</span> <span class="pre">explore'</span></code> fails.</li>
</ul>
</div>
</div>
<div class="section" id="release-3-2-0">
<h2><a class="reference external" href="http://docs.cask.co/cdap/3.2.0/index.html">Release 3.2.0</a><a class="headerlink" href="#release-3-2-0" title="Permalink to this headline">🔗</a></h2>
<div class="section" id="id105">
<h3>New Features<a class="headerlink" href="#id105" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2556">CDAP-2556</a> -
Added support for HBase1.1.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2666">CDAP-2666</a> -
Added a new API for creating an application from an artifact.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2756">CDAP-2756</a> -
Added the ability to write to multiple outputs from a MapReduce job.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2757">CDAP-2757</a> -
Added the ability to dynamically write to multiple partitions of a PartitionedFileSet
dataset as the output of a MapReduce job.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3253">CDAP-3253</a> -
Added a Stream and Dataset Widget to the CDAP UI.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3390">CDAP-3390</a> -
Added stream views, enabling reading from a single stream using various formats and
schemas.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3476">CDAP-3476</a> -
Added a Validator Transform that can be used to validate records based on a set of
available validators and configured to write invalid records to an error
dataset.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3516">CDAP-3516</a> -
Added a service to manage the metadata of CDAP entities.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3518">CDAP-3518</a> -
Added the publishing of metadata change notifications to Apache Kafka.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3519">CDAP-3519</a> -
Added the ability to compute lineage of a CDAP dataset or stream in a given time window.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3520">CDAP-3520</a> -
Added RESTful APIs for adding/retrieving/deleting of metadata for apps/programs/datasets/streams.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3521">CDAP-3521</a> -
Added the ability to record a dataset or stream access by a CDAP program.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3522">CDAP-3522</a> -
Added the capability to search CDAP entities based on their metadata.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3523">CDAP-3523</a> -
Added RESTful APIs for searching CDAP entities based on business metadata.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3527">CDAP-3527</a> -
Added a data store to manage business metadata of CDAP entities.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3549">CDAP-3549</a> -
Added SSH port forwarding to the CDAP virtual machine.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3556">CDAP-3556</a> -
Added a data store for recording data accesses by CDAP programs and computing lineage.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3590">CDAP-3590</a> -
Added the ability to write to multiple sinks in ETL real-time and batch applications.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3591">CDAP-3591</a> -
Added the ability for real-time ETL pipelines to write to multiple sinks.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3592">CDAP-3592</a> -
Added the ability for batch ETL pipelines to write to multiple sinks.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3626">CDAP-3626</a> -
For the CSV and TSV stream formats, a “mapping” setting can now be specified, mapping
stream event columns to schema columns.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3693">CDAP-3693</a> -
Added support for CDAP to work with HDP 2.3.</li>
</ul>
</div>
<div class="section" id="id106">
<h3>Improvements<a class="headerlink" href="#id106" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-1914">CDAP-1914</a> -
Added documentation of the RESTful endpoint to retrieve the properties of a stream.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2514">CDAP-2514</a> -
Added an interface to load a file into a stream from the CDAP UI.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2809">CDAP-2809</a> -
The CDAP UI “Errors” pop-up in the main screen now displays the time and date for each
error.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2872">CDAP-2872</a> -
Updated the Cloudera Manager CSD to use support for logback.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2950">CDAP-2950</a> -
Cleaned up the messages shown in the errors dropdown in the CDAP UI.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3147">CDAP-3147</a> -
Added a CDAP CLI command to stop a workflow.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3179">CDAP-3179</a> -
Added support for upgrading the Hadoop distribution or the HBase version that CDAP is
running on.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3257">CDAP-3257</a> -
Revised the documentation of the file <code class="docutils literal notranslate"><span class="pre">cdap-default.xml</span></code>, removed properties no longer
in use, and corrected discrepancies between the documentation and the shipped XML
file.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3270">CDAP-3270</a> -
Improved the help provided in the CDAP CLI for the setting of stream formats.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3275">CDAP-3275</a> -
Upgraded netty-http version to 0.12.0.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3282">CDAP-3282</a> -
Added a HTTP RESTful API to update the application configuration and artifact version.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3332">CDAP-3332</a> -
Added a “clear” button in the CDAP UI for cases where a user decides to not used a
pre-populated schema.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3351">CDAP-3351</a> -
Defined a directory structure to be used for predefined applications.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3357">CDAP-3357</a> -
Added documentation in the source code on adding new commands and completers to the CDAP CLI.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3393">CDAP-3393</a> -
In the CDAP UI, added visualization for Workflow tokens in Workflows.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3419">CDAP-3419</a> -
HBaseQueueDebugger now shows the minimum queue event transaction write pointer both for
each queue and for all queues.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3443">CDAP-3443</a> -
Added an example cdap-env.sh to the shipped packages.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3464">CDAP-3464</a> -
Added an example in the documentation explaining how to prune invalid transactions from
the transaction manager.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3490">CDAP-3490</a> -
Modified the CDAP upgrade tool to delete all adapters and the ETLBatch and ETLRealtime
ApplicationTemplates.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3495">CDAP-3495</a> -
Added the ability to persist the runtime arguments with which a program was run.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3550">CDAP-3550</a> -
Added support for writing to Amazon S3 in Avro and Parquet formats from batch ETL
applications.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3564">CDAP-3564</a> -
Updated CDAP to use Tephra 0.6.2.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3610">CDAP-3610</a> -
Updated the transaction debugger client to print checkpoint information.</li>
</ul>
</div>
<div class="section" id="id107">
<h3>Bug Fixes<a class="headerlink" href="#id107" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-1697">CDAP-1697</a> -
Fixed an issue where failed dataset operations via Explore queries did not invalidate
the associated transaction.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-1864">CDAP-1864</a> -
Fixed a problem where users got an incorrect message while creating a dataset in a
non-existent namespace.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-1892">CDAP-1892</a> -
Fixed a problem with services returning the same message for all failures.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-1984">CDAP-1984</a> -
Fixed a problem where a dataset could be created in a non-existent namespace in
standalone mode.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2428">CDAP-2428</a> -
Fixed a problem with the CDAP CLI creating file logs.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2521">CDAP-2521</a> -
Fixed a problem with the CDAP CLI not auto-completing when setting a stream format.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2785">CDAP-2785</a> -
Fixed a problem with the CDAP UI of buttons staying ‘in focus’ after clicking.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2809">CDAP-2809</a> -
The CDAP UI “Errors” pop-up in the main screen now displays the time and date for each error.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2892">CDAP-2892</a> -
Fixed a problem with schedules not being deployed in suspended mode.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3014">CDAP-3014</a> -
Fixed a problem where failure of a spark node would cause a workflow to restart indefinitely.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3073">CDAP-3073</a> -
Fixed an issue with the Standalone CDAP process periodically crashing with Out-of-Memory
errors when writing to an Oracle table.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3101">CDAP-3101</a> -
Fixed a problem with workflow runs not getting scheduled due to Quartz exceptions.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3121">CDAP-3121</a> -
Fixed a problem with discrepancies between the documentation and the defaults actually used by CDAP.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3200">CDAP-3200</a> -
Fixed a problem in the CDAP UI with the clone button in an incorrect position when using Firefox.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3201">CDAP-3201</a> -
Fixed a problem in the CDAP UI with an incorrect tabbing order when using Firefox.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3219">CDAP-3219</a> -
Fixed a problem when specifying the HBase version using the HBASE_VERSION environment variable.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3233">CDAP-3233</a> -
Fixed a problem in the CDAP UI error pop-ups not having a default focus on a button.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3243">CDAP-3243</a> -
Fixed a problem in the CDAP UI with the default schema shown for streams.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3260">CDAP-3260</a> -
Fixed a problem in the CDAP UI with scrolling on the namespaces dropdown on certain pages.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3261">CDAP-3261</a> -
Fixed a problem on Distributed CDAP with the serializing of the metadata artifact
causing a stack overflow.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3305">CDAP-3305</a> -
Fixed a problem in the CDAP UI not warning users if they exit or close their browser without saving.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3313">CDAP-3313</a> -
Fixed a problem in the CDAP UI with refreshing always returning to the overview page.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3326">CDAP-3326</a> -
Fixed a problem with the table batch source requiring a row key to be set.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3343">CDAP-3343</a> -
Fixed a problem with the application deployment for apps that contain Spark.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3349">CDAP-3349</a> -
Fixed a problem with the display of ETL application metrics in the CDAP UI.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3355">CDAP-3355</a> -
Fixed a problem in the CDAP examples with the use of a runtime argument, <code class="docutils literal notranslate"><span class="pre">min.pages.threshold</span></code>.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3362">CDAP-3362</a> -
Fixed a problem with the <code class="docutils literal notranslate"><span class="pre">logback-container.xml</span></code> not being copied into master services.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3374">CDAP-3374</a> -
Fixed a problem with warning messages in the logs indicating that programs were running
that actually were not running.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3376">CDAP-3376</a> -
Fixed a problem with being unable to deploy the SparkPageRank example application on a cluster.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3386">CDAP-3386</a> -
Fixed a problem with the Spark classes not being found when running a Spark program
through a Workflow in Distributed CDAP on HDP 2.2.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3394">CDAP-3394</a> -
Fixed a problem with the deployment of applications through the CDAP UI.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3399">CDAP-3399</a> -
Fixed a problem with the SparkPageRankApp example spawning multiple containers in
distributed mode due to its number of services.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3400">CDAP-3400</a> -
Fixed an issue with warning messages about the notification system every time the CDAP
Standalone is restarted.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3408">CDAP-3408</a> -
Fixed a problem with running the CDAP Explore Service on CDH 5.[2,3].</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3432">CDAP-3432</a> -
Fixed a bug where connecting with a certain namespace from the CLI would not immediately
display that namespace in the CLI prompt.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3435">CDAP-3435</a> -
Fixed an issue where the program status was shown as running even after it is stopped.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3442">CDAP-3442</a> -
Fixed a problem that caused application creation to fail if a config setting was given
to an application that does not use a config.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3449">CDAP-3449</a> -
Fixed a problem with the readless increment co-processor not handling multiple readless
increment columns in the same row.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3452">CDAP-3452</a> -
Fixed a problem that prevented explore service working on clusters with secure hive 0.14.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3458">CDAP-3458</a> -
Fixed a problem where streams events that had already been processed were re-processed in flows.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3470">CDAP-3470</a> -
Fixed an issue with error messages being logged during a master process restart.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3472">CDAP-3472</a> -
Fixed the error message returned when trying to stop a program started by a workflow.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3473">CDAP-3473</a> -
Fixed a problem with a workflow failure not updating a run record for the inner program.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3530">CDAP-3530</a> -
Fixed a problem with the CDAP UI performance when rendering flow diagrams with a large number of nodes.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3563">CDAP-3563</a> -
Removed faulty and unused metrics around CDAP file resource usage.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3574">CDAP-3574</a> -
Fix an issue with Explore not working on HDP Hive 0.12.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3603">CDAP-3603</a> -
Fixed an issue with configuration properties for ETL Transforms being validated at
runtime instead of when an application is created.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3618">CDAP-3618</a> -
Fix a problem where suspended schedules were lost when CDAP master was restarted.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3660">CDAP-3660</a> -
Fixed and issue where the Hadoop filesystem object was getting instantiated before the
Kerberos keytab login was completed, leading to CDAP processes failing after the initial
ticket expired.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3700">CDAP-3700</a> -
Fixed an issue with the log saver having numerous open connections to HBase, causing it
to go Out-of-Memory.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3711">CDAP-3711</a> -
Fixed an issue that prevented the downloading of Explore results on a secure cluster.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3713">CDAP-3713</a> -
Fixed an issue where certain RESTful APIs were not returning appropriate error messages
for internal server errors.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3716">CDAP-3716</a> -
Fixed a possible deadlock when CDAP master is restarted with an existing app running on a cluster.</li>
</ul>
</div>
<div class="section" id="id108">
<h3>API Changes<a class="headerlink" href="#id108" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2763">CDAP-2763</a> -
Added RESTful APIs for managing artifacts.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2956">CDAP-2956</a> -
Deprecated the existing API for configuring a workflow action, replacing it with a
simpler API.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3063">CDAP-3063</a> -
Added CLI commands for managing artifacts.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3064">CDAP-3064</a> -
Added an ArtifactClient to interact with Artifact HTTP RESTful APIs.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3283">CDAP-3283</a> -
Added artifact information to Application RESTful APIs and the means to filter
applications by artifact name and version.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3324">CDAP-3324</a> -
Added a RESTful API for creating an application from an artifact.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3367">CDAP-3367</a> -
Added the ability to delete an artifact.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3488">CDAP-3488</a> -
Changed the ETLBatchTemplate from an ApplicationTemplate to an Application.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3535">CDAP-3535</a> -
Added an API for programs to retrieve their application specification at runtime.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3554">CDAP-3554</a> -
Changed the plugin types from ‘source’ to either ‘batchsource’ or ‘realtimesource’, and
from ‘sink’ to either ‘batchsink’ or ‘realtimesink’ to reflect that the plugins
implement different interfaces.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-1554">CDAP-1554</a> -
Moved constants for default and system namespaces from Common to Id.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3388">CDAP-3388</a> -
Added interfaces to <code class="docutils literal notranslate"><span class="pre">cdap-spi</span></code> that abstract StreamEventRecordFormat (and dependent
interfaces) so users can extend the <code class="docutils literal notranslate"><span class="pre">cdap-spi</span></code> interfaces.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3583">CDAP-3583</a> -
Added a RESTful API for retrieving the metadata associated with a particular run of a
CDAP program.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3632">CDAP-3632</a> -
Added a RESTful API for computing lineage of a CDAP dataset or stream.</li>
</ul>
</div>
<div class="section" id="id109">
<h3>Deprecated and Removed Features<a class="headerlink" href="#id109" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li>See the <a class="reference external" href="http://docs.cask.co/cdap/3.2.0/en/reference-manual/javadocs/index.html#javadocs">CDAP 3.2.0 Javadocs</a>
for a list of deprecated and removed APIs.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2667">CDAP-2667</a> -
Removed application templates and adapters RESTful APIs, as these templates and adapters
have been replaced with applications that can be controlled with the
<a class="reference external" href="http://docs.cask.co/cdap/3.2.0/en/reference-manual/http-restful-api/lifecycle.html#http-restful-api-lifecycle">Lifecycle HTTP RESTful API</a>.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2951">CDAP-2951</a> -
Removed deprecated methods in cdap-client.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3596">CDAP-3596</a> -
Replaced the ETL ApplicationTemplates with the new ETL Applications.</li>
</ul>
</div>
<div class="section" id="id110">
<h3>Known Issues<a class="headerlink" href="#id110" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li>After upgrading CDAP from a pre-3.0 version, any unprocessed metrics data in Kafka will
be lost and <em>WARN</em> log messages will be logged that tell about the inability to process
old data in the old format.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-797">CDAP-797</a> -
When running secure Hadoop clusters, debug logs from MapReduce programs are not
available.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-1007">CDAP-1007</a> -
If the Hive Metastore is restarted while the CDAP Explore Service is running, the
Explore Service remains alive, but becomes unusable. To correct, restart the CDAP Master—which will restart all services—as described under “Starting CDAP Services”
for your particular Hadoop distribution in the <a class="reference external" href="http://docs.cask.co/cdap/3.2.0/en/admin-manual/installation/index.html#installation-index">Installation documentation</a>.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-1587">CDAP-1587</a> -
CDAP internally creates tables in the “user” space that begin with the word
<code class="docutils literal notranslate"><span class="pre">&quot;system&quot;</span></code>. User datasets with names starting with <code class="docutils literal notranslate"><span class="pre">&quot;system&quot;</span></code> can conflict if they
were to match one of those names. To avoid this, do not start any datasets with the word
<code class="docutils literal notranslate"><span class="pre">&quot;system&quot;</span></code>.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2632">CDAP-2632</a> -
The application in the <a class="reference external" href="https://github.com/cdap-guides/cdap-kafka-ingest-guide/tree/release/cdap-3.0-compatible">cdap-kafka-ingest-guide</a>
does not run on Ubuntu 14.x as of CDAP 3.0.x.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2721">CDAP-2721</a> -
Metrics for <a class="reference external" href="http://docs.cask.co/cdap/3.2.0/en/developers-manual/building-blocks/datasets/fileset.html#datasets-fileset">FileSets</a>
can show zero values even if there is
data present, because FileSets do not emit metrics (<cite>CDAP-587
&lt;https://issues.cask.co/browse/CDAP-587&gt;</cite>).</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2831">CDAP-2831</a> -
A workflow that is scheduled by time will not be run between the failure of the primary
master and the time that the secondary takes over. This scheduled run will not be
triggered at all.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2945">CDAP-2945</a> -
If the input partition filter for a PartitionedFileSet does not match any partitions,
MapReduce jobs can fail.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3000">CDAP-3000</a> -
The Workflow token is in an inconsistent state for nodes in a fork while the nodes of
the fork are still running. It becomes consistent after the join.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3221">CDAP-3221</a> -
When running in Standalone CDAP, if a MapReduce job fails repeatedly, then the SDK
hits an out-of-memory exception due to <code class="docutils literal notranslate"><span class="pre">perm</span> <span class="pre">gen</span></code>. The Standalone needs restarting at
this point.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3262">CDAP-3262</a> -
For Microsoft Windows, the Standalone CDAP scripts can fail when used with a JAVA_HOME
that is defined as a path with spaces in it. A workaround is to use a definition of
JAVA_HOME that does not include spaces, such as <code class="docutils literal notranslate"><span class="pre">C:\PROGRA~1\Java\jdk1.7.0_79\bin</span></code> or
<code class="docutils literal notranslate"><span class="pre">C:\ProgramData\Oracle\Java\javapath</span></code>.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3492">CDAP-3492</a> -
In the CDAP CLI, executing <code class="docutils literal notranslate"><span class="pre">select</span> <span class="pre">*</span></code> from a dataset with many fields generates an error.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3641">CDAP-3641</a> -
A RESTful API call to retrieve workflow statistics hangs if units (such as “s” for
seconds) are not provided as part of the query.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3697">CDAP-3697</a> -
CDAP Explore is broken on secure CDH 5.1.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3698">CDAP-3698</a> -
CDAP Explore is unable to get a delegation token while fetching next results on HDP2.0.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3749">CDAP-3749</a> -
The DBSource plugin does not allow a username with an empty password.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3750">CDAP-3750</a> -
If a table schema contains a field name that is a reserved word in the Hive DDL, <code class="docutils literal notranslate"><span class="pre">'enable</span> <span class="pre">explore'</span></code> fails.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3819">CDAP-3819</a> -
The Cassandra source does not handles spaces properly in column fields which require a comma-separated list.</li>
</ul>
</div>
</div>
<div class="section" id="release-3-1-0">
<h2><a class="reference external" href="http://docs.cask.co/cdap/3.1.0/index.html">Release 3.1.0</a><a class="headerlink" href="#release-3-1-0" title="Permalink to this headline">🔗</a></h2>
<div class="section" id="id111">
<h3>New Features<a class="headerlink" href="#id111" title="Permalink to this headline">🔗</a></h3>
<p><strong>MapR 4.1 Support, HDP 2.2 Support, CDH 5.4 Support</strong></p>
<ul class="simple">
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-1614">CDAP-1614</a> -
Added HBase 1.0 support.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2318">CDAP-2318</a> -
Made CDAP work on the HDP 2.2 distribution.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2786">CDAP-2786</a> -
Added support to CDAP 3.1.0 for the MapR 4.1 distro.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2798">CDAP-2798</a> -
Added Hive 0.14 support.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2801">CDAP-2801</a> -
Added CDH 5.4 Hive 1.1 support.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2836">CDAP-2836</a> -
Added support for restart of specific CDAP System Services Instances.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2853">CDAP-2853</a> -
Completed certification process for MapR on CDAP.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2879">CDAP-2879</a> -
Added Hive 1.0 in Standalone.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2881">CDAP-2881</a> -
Added support for HDP 2.2.x.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2891">CDAP-2891</a> -
Documented cdap-env.sh and settings OPTS for HDP 2.2.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2898">CDAP-2898</a> -
Added Hive 1.1 in Standalone.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2953">CDAP-2953</a> -
Added HiveServer2 support in a secure cluster.</li>
</ul>
<p><strong>Spark</strong></p>
<ul class="simple">
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-344">CDAP-344</a> -
Users can now run Spark in distributed mode.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-1993">CDAP-1993</a> -
Added ability to manipulate the SparkConf.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2700">CDAP-2700</a> -
Added the ability to Spark programs of discovering CDAP services in distributed mode.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2701">CDAP-2701</a> -
Spark programs are able to collect Metrics in distributed mode.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2703">CDAP-2703</a> -
Users are able to collect/view logs from Spark programs in distributed mode.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2705">CDAP-2705</a> -
Added examples, guides and documentation for Spark in distributed mode. LogAnalysis
application demonstrating parallel execution of the Spark and MapReduce programs using
Workflows.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2923">CDAP-2923</a> -
Added support for the WorkflowToken in the Spark programs.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2936">CDAP-2936</a> -
Spark program can now specify resources usage for driver and executor process in distributed mode.</li>
</ul>
<p><strong>Workflows</strong></p>
<ul class="simple">
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-1983">CDAP-1983</a> -
Added example application for processing and analyzing Wikipedia data using Workflows.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2709">CDAP-2709</a> -
Added ability to add generic keys to the WorkflowToken.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2712">CDAP-2712</a> -
Added ability to update the WorkflowToken in MapReduce and Spark programs.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2713">CDAP-2713</a> -
Added ability to persist the WorkflowToken per run of the Workflow.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2714">CDAP-2714</a> -
Added ability to query the WorkflowToken for the past as well as currently running Workflow runs.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2752">CDAP-2752</a> -
Added ability for custom actions to access the CDAP datasets and services.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2894">CDAP-2894</a> -
Added an API to retreive the system properties (e.g. MapReduce counters in case of
MapReduce program) from the WorkflowToken.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2923">CDAP-2923</a> -
Added support for the WorkflowToken in the Spark programs.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2982">CDAP-2982</a> -
Added verification that the Workflow contains all programs/custom actions with a unique name.</li>
</ul>
<p><strong>Datasets</strong></p>
<ul>
<li><p class="first"><a class="reference external" href="https://issues.cask.co/browse/CDAP-347">CDAP-347</a> -
User can use datasets in beforeSubmit and afterFinish.</p>
</li>
<li><p class="first"><a class="reference external" href="https://issues.cask.co/browse/CDAP-585">CDAP-585</a> -
Changes to Spark program runner to use File dataset in Spark.
Spark programs can now use file-based datasets.</p>
</li>
<li><p class="first"><a class="reference external" href="https://issues.cask.co/browse/CDAP-2734">CDAP-2734</a> -
Added PartitionedFileSet support to setting/getting properties at the Partition level.</p>
</li>
<li><p class="first"><a class="reference external" href="https://issues.cask.co/browse/CDAP-2746">CDAP-2746</a> -
PartitionedFileSets now record the creation time of each partition in the metadata.</p>
</li>
<li><p class="first"><a class="reference external" href="https://issues.cask.co/browse/CDAP-2747">CDAP-2747</a> -
PartitionedFileSets now index the creation time of partitions to allow selection of
partitions that were created after a given time. Introduced BatchPartitionConsumer as a
way to incrementally consume new data in a PartitionedFileSet.</p>
</li>
<li><p class="first"><a class="reference external" href="https://issues.cask.co/browse/CDAP-2752">CDAP-2752</a> -
Added ability for custom actions to access the CDAP datasets and services.</p>
</li>
<li><p class="first"><a class="reference external" href="https://issues.cask.co/browse/CDAP-2758">CDAP-2758</a> -
FileSet now support existing HDFS locations.</p>
<p>Treat base paths that start with “/” as absolute in the file system. An absolute base
path for a (Partitioned)FileSet was interpreted as relative to the namespace’s data
directory. Newly created FileSets interpret absolute base paths as absolute in the file
system.</p>
<p>Introduced a new property for (Partitioned)FileSets name “data.external”. If true, the
base path of the FileSet is assumed to be managed by some external process. That is, the
FileSet will not attempt to create the directory, it will not delete any files when the
FileSet is dropped or truncated, and it will not allow adding or deleting files or
partitions. In other words, the FileSet is read-only.</p>
</li>
<li><p class="first"><a class="reference external" href="https://issues.cask.co/browse/CDAP-2784">CDAP-2784</a> -
Added support to write to PartitionedFileSet Partition metadata from MapReduce.</p>
</li>
<li><p class="first"><a class="reference external" href="https://issues.cask.co/browse/CDAP-2822">CDAP-2822</a> -
IndexedTable now supports scans on the indexed field.</p>
</li>
</ul>
<p><strong>Metrics</strong></p>
<ul class="simple">
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2975">CDAP-2975</a> -
Added pre-split FactTables.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2326">CDAP-2326</a> -
Added better unit-test coverage for Cube dataset.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-1853">CDAP-1853</a> -
Metrics processor scaling no longer needs a master services restart.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2844">CDAP-2844</a> -
MapReduce metrics collection no longer use counters, and instead report directly to Kafka.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2701">CDAP-2701</a> -
Spark programs are able to collect Metrics in distributed mode.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2466">CDAP-2466</a> -
Added CLI for metrics search and query.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2236">CDAP-2236</a> -
New CDAP UI switched over to using newer search/query APIs.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-1998">CDAP-1998</a> -
Removed deprecated Context - Query param in Metrics v3 API.</li>
</ul>
<p><strong>Miscellaneous New Features</strong></p>
<ul class="simple">
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-332">CDAP-332</a> -
Added a Restful end-point for deleting Streams.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-1483">CDAP-1483</a> -
QueueAdmin now uses Id.Namespace instead of simply String.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-1584">CDAP-1584</a> -
CDAP CLI now shows the username in the CLI prompt.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2139">CDAP-2139</a> -
Removed a duplicate Table of Contents on the Documentation Search page.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2515">CDAP-2515</a> -
Added a metrics client for search and query by tags.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2582">CDAP-2582</a> -
Documented the licenses of the shipped CDAP UI components.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2595">CDAP-2595</a> -
Added data modelling of flows.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2596">CDAP-2596</a> -
Added data modelling of MapReduce.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2617">CDAP-2617</a> -
Added the capability to get logs for a given time range from CLI.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2618">CDAP-2618</a> -
Simplified the Cube sink configurations.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2670">CDAP-2670</a> -
Added Parquet sink with time partitioned file dataset.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2739">CDAP-2739</a> -
Added S3 batch source for ETLbatch.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2802">CDAP-2802</a> -
Stopped using HiveConf.ConfVars.defaultValue, to support Hive &gt;0.13.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2847">CDAP-2847</a> -
Added ability to add custom filters to FileBatchSource.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2893">CDAP-2893</a> -
Custom Transform now parses log formats for ETL.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2913">CDAP-2913</a> -
Provided installation method for EMR.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2915">CDAP-2915</a> -
Added an SQS real-time plugin for ETL.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3022">CDAP-3022</a> -
Added Cloudfront format option to LogParserTransform.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3032">CDAP-3032</a> -
Documented TestConfiguration class usage in unit-test framework.</li>
</ul>
</div>
<div class="section" id="id112">
<h3>Improvements<a class="headerlink" href="#id112" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-593">CDAP-593</a> -
Spark no longer determines the mode through MRConfig.FRAMEWORK_NAME.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-595">CDAP-595</a> -
Refactored SparkRuntimeService and SparkProgramWrapper.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-665">CDAP-665</a> -
Documentation received a product-specifc 404 Page.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-683">CDAP-683</a> -
Changed all README files from markdown to rst format.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-1132">CDAP-1132</a> -
Improved the CDAP Doc Search Result Sorting.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-1416">CDAP-1416</a> -
Added links to upper level pages on Docs.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-1572">CDAP-1572</a> -
Standardized Id classes.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-1583">CDAP-1583</a> -
Refactored InMemoryWorkerRunner and ServiceProgramRunnner after ServiceWorkers were removed.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-1918">CDAP-1918</a> -
Switched to using the Spark 1.3.0 release.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-1926">CDAP-1926</a> -
Streams endpoint accept “now”, “now-30s”, etc., for time ranges.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2007">CDAP-2007</a> -
CLI output for “call service” is rendered in a copy-pastable manner.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2310">CDAP-2310</a> -
Kafka Source now able to apply a Schema to the Payload received.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2388">CDAP-2388</a> -
Added Java 8 support to CDAP.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2422">CDAP-2422</a> -
Removed redundant catch blocks in AdapterHttpHandler.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2455">CDAP-2455</a> -
Version in CDAP UI footer is dynamic.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2482">CDAP-2482</a> -
Reduced excessive capitalisation in documentation.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2531">CDAP-2531</a> -
Adapter details made available through CDAP UI.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2539">CDAP-2539</a> -
Added a build identifier (branch, commit) in header of Documentation HTML pages.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2552">CDAP-2552</a> -
Documentation Build script now flags errors.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2554">CDAP-2554</a> -
Documented that streams can now be deleted.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2557">CDAP-2557</a> -
Non-handler logic moved out of DatasetInstanceHandler.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2570">CDAP-2570</a> -
CLI prompt changes to ‘DISCONNECTED’ after CDAP is stopped.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2578">CDAP-2578</a> -
Ability to look at configs of created adapters.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2585">CDAP-2585</a> -
Use Id in cdap-client rather than Id.Namespace + String.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2588">CDAP-2588</a> -
Improvements to the MetricsClient APIs.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2590">CDAP-2590</a> -
Switching namespaces when in CDAP UI Operations screens.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2620">CDAP-2620</a> -
CDAP clients now use Id classes from cdap proto, instead of plain strings.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2628">CDAP-2628</a> -
CDAP UI: Breadcrumbs in Workflow/Mapreduce work as expected.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2644">CDAP-2644</a> -
In cdap-clients, no longer need to retrieve runtime arguments before starting a program.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2651">CDAP-2651</a> -
CDAP UI: the Namespace is made more prominent.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2681">CDAP-2681</a> -
CDAP UI: scrolling no longer enlarges the workflow diagram instead of scrolling through.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2683">CDAP-2683</a> -
CDAP UI: added a remove icons for fork and Join.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2684">CDAP-2684</a> -
CDAP UI: workflow diagrams are directed graphs.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2688">CDAP-2688</a> -
CDAP UI: added search &amp; pagination for lists of apps and datasets.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2689">CDAP-2689</a> -
CDAP UI: shows which application is a part of which dataset.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2691">CDAP-2691</a> -
CDAP UI: added ability to delete streams.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2692">CDAP-2692</a> -
CDAP UI: added pagination for logs.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2694">CDAP-2694</a> -
CDAP UI: added a loading icon/UI element when creating an adapter.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2695">CDAP-2695</a> -
CDAP UI: long names of adapters are replaced by a short version ending in an ellipsis.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2697">CDAP-2697</a> -
CDAP UI: added tab names during adapter creation.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2716">CDAP-2716</a> -
CDAP UI: when creating an adapter, the tabbing order shows correctly.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2733">CDAP-2733</a> -
Implemented a TimeParitionedFileSet source.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2811">CDAP-2811</a> -
Improved Hive version detection.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2921">CDAP-2921</a> -
Removed backward-compatibility for pre-2.8 TPFS.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2938">CDAP-2938</a> -
Implemented new ETL application template creation.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2983">CDAP-2983</a> -
Spark program runner now calls onFailure() of the DatasetOutputCommitter.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2986">CDAP-2986</a> -
Spark program now are able to specify runtime arguments when reading or writing a datset.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2987">CDAP-2987</a> -
Added an example for Spark using datasets directly.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2989">CDAP-2989</a> -
Added an example for Spark using FileSets.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3018">CDAP-3018</a> -
Updated workflow guides for workflow token.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3028">CDAP-3028</a> -
Improved the system service restart endpoint to handle illegal instance IDs and “service not available”.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3053">CDAP-3053</a> -
Added schema javadocs that explain how to write the schema to JSON.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3077">CDAP-3077</a> -
Add the ability in TableSink to find schema.row.field case-insensitively.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3144">CDAP-3144</a> -
Changed CLI command descriptions to use consistent element case.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3152">CDAP-3152</a> -
Refactored ETLBatch sources and sinks.</li>
</ul>
</div>
<div class="section" id="id113">
<h3>Bug Fixes<a class="headerlink" href="#id113" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-23">CDAP-23</a> -
Fixed a problem with the DatasetFramework not loading a given dataset with the same classloader across calls.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-68">CDAP-68</a> -
Made sure all network services in Singlenode only bind to localhost.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-376">CDAP-376</a> -
Fixed a problem with HBaseOrderedTable never calling HTable.close().</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-550">CDAP-550</a> -
Consolidated Examples, Guides, and Tutorials styles.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-598">CDAP-598</a> -
Fixed problems with the CDAP ClassLoading model.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-674">CDAP-674</a> -
Fixed problems with CDAP code examples and versioning.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-814">CDAP-814</a> -
Resolved issues in the documentation about element versus program.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-1042">CDAP-1042</a> -
Fixed a problem with specifying dataset selection as input for Spark job.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-1145">CDAP-1145</a> -
Fixed the PurchaseAppTest.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-1184">CDAP-1184</a> -
Fixed a problem with the DELETE call not clearing queue metrics.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-1273">CDAP-1273</a> -
Fixed a problem with the ProgramClassLoader getResource.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-1457">CDAP-1457</a> -
Fixed a memory leak of user class after running Spark program.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-1552">CDAP-1552</a> -
Fixed a problem with Mapreduce progress metrics not being interpolated.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-1868">CDAP-1868</a> -
Fixed a problem with Java Client and CLI not setting set dataset properties on existing datasets.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-1873">CDAP-1873</a> -
Fixed a problem with warnings and errors when CDAP-Master starts up.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-1967">CDAP-1967</a> -
Fixed a problem with CDAP-Master failing to start up due to conflicting dependencies.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-1976">CDAP-1976</a> -
Fixed a problem with examples not following the same pattern.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-1988">CDAP-1988</a> -
Fixed a problem with creating a Dataset through REST API failing if no properties are provided.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2081">CDAP-2081</a> -
Fixed a problem with StreamSizeSchedulerTest failing randomly.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2140">CDAP-2140</a> -
Fixed a problem with the CDAP UI not showing system service status when system services are down.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2177">CDAP-2177</a> -
Fixed a problem with Enable and Fix LogSaverPluginTest.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2208">CDAP-2208</a> -
Fixed a problem with CDAP-Explore service failing on wrapped indexedTable with Avro (specific record) contents.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2228">CDAP-2228</a> -
Fixed a problem with Mapreduce not working in Hadoop 2.2.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2254">CDAP-2254</a> -
Fixed a problem with an incorrect error message returned by HTTP RESTful Handler.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2258">CDAP-2258</a> -
Fixed a problem with an internal error when attempting to start a non-existing program.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2279">CDAP-2279</a> -
Fixed a problem with namespace and gear widgets disappearing when the browser window is too narrow.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2280">CDAP-2280</a> -
Fixed a problem when starting a flow from the GUI that the GUI does not fully refresh the page.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2341">CDAP-2341</a> -
Fixed a problem that when a MapReduce fails to start, it cannot be started or stopped any more.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2343">CDAP-2343</a> -
Fixed a problem in the CDAP UI that Mapreduce logs are convoluted with system logs.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2344">CDAP-2344</a> -
Fixed a problem with the formatting of logs in the CDAP UI.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2355">CDAP-2355</a> -
Fixed a problem with an Adapter CLI help error.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2356">CDAP-2356</a> -
Fixed a problem with CLI autocompletion results not sorted in alphabetical order.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2365">CDAP-2365</a> -
Fixed a problem that when restarting CDAP-Master, the CDAP UI oscillates between being up and down.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2376">CDAP-2376</a> -
Fixed a problem with logs from mapper and reducer not being collected.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2444">CDAP-2444</a> -
Fixed a problem with Cloudera Configuring doc needs fixing.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2446">CDAP-2446</a> -
Fixed a problem with that examples needing to be updated for new CDAP UI.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2454">CDAP-2454</a> -
Fixed a problem with Proto class RunRecord containing the Apache Twill RunId when serialized in REST API response.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2459">CDAP-2459</a> -
Fixed a problem with the CDAP UI going into a loop when the Router returns 200 and App Fabric is not up.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2474">CDAP-2474</a> -
Fixed a problem with changing the format of the name for the connectionfactory in JMS source plugin.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2475">CDAP-2475</a> -
Fixed a problem with JMS source accepting the type and name of the JMS provider plugin.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2480">CDAP-2480</a> -
Fixed a problem with the Workflow current run info endpoint missing a /runs/ in the path.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2489">CDAP-2489</a> -
Fixed a problem when, in distributed mode and CDAP master restarted, status of the running PROGRAM is always returned as STOPPED.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2490">CDAP-2490</a> -
Fixed a problem with checking if invalid Run Records for Spark and MapReduce are part of run from Workflow child programs.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2491">CDAP-2491</a> -
Fixed a problem with the MapReduce program in standalone mode not always using LocalJobRunnerWithFix.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2493">CDAP-2493</a> -
After the fix for <a class="reference external" href="https://issues.cask.co/browse/CDAP-2474">CDAP-2474</a> (ConnectionFactory in JMS source),
the JSON file requires updating for the change to reflect in CDAP UI.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2496">CDAP-2496</a> -
Fixed a problem with CDAP using its own transaction snapshot codec.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2498">CDAP-2498</a> -
Fixed a problem with validation while creating adapters only by types and not also by values.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2517">CDAP-2517</a> -
Fixed a problem with Explore docs not mentioning partitioned file sets.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2520">CDAP-2520</a> -
Fixed a problem with StreamSource not liking values of ‘0m’.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2522">CDAP-2522</a> -
Fixed a problem with TransactionStateCache needing to reference Tephra SnapshotCodecV3.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2529">CDAP-2529</a> -
Fixed a problem with CLI not printing an error if it can’t connect to CDAP.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2530">CDAP-2530</a> -
Fixed a problem with Custom RecordScannable&lt;StructuredRecord&gt; datasets not be explorable.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2535">CDAP-2535</a> -
Fixed a problem with IntegrationTestManager deployApplication not being namespaced.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2538">CDAP-2538</a> -
Fixed a problem with handling extra whitespace in CLI input.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2540">CDAP-2540</a> -
Fixed a problem with the Preferences Namespace CLI help having errors.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2541">CDAP-2541</a> -
Added the ability to stop the particular run of a program. Allows concurrent runs of the
MapReduce and Workflow programs and the ability to stop programs at a per-run level.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2547">CDAP-2547</a> -
Fixed a problem with Kakfa Source - not using the persisted offset when the Adapter is restarted.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2549">CDAP-2549</a> -
Fixed a problem with a suspended workflow run record not being removed upon app/namespace delete.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2562">CDAP-2562</a> -
Fixed a problem with the automated Doc Build failing in develop.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2564">CDAP-2564</a> -
Improved the management of dataset resources.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2565">CDAP-2565</a> -
Fixed a problem with the transaction latency metric being of incorrect type.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2569">CDAP-2569</a> -
Fixed a problem with master process not being resilient to zookeeper exceptions.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2571">CDAP-2571</a> -
Fixed a problem with the RunRecord thread not resilient to errors.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2587">CDAP-2587</a> -
Fixed a problem with being unable to create default namespaces on starting up SDK.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2635">CDAP-2635</a> -
Fixed a problem with Namespace Create ignoring the properties’ config field.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2636">CDAP-2636</a> -
Fixed a problem with “out of perm gen” space in CDAP Explore service.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2654">CDAP-2654</a> -
Fixed a problem with False values showing up as ‘false null’ in the CDAP Explore UI.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2685">CDAP-2685</a> -
Fixed a problem with the CDAP UI: no empty box for transforms.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2729">CDAP-2729</a> -
Fixed a problem with CDAP UI not handling downstream system services gracefully.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2740">CDAP-2740</a> -
Fixed a problem with CDAP UI not gracefully handling when the nodejs server goes down.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2748">CDAP-2748</a> -
Fixed a problem with the currently running and completed status of Spark programs in a
workflow not highlighted in the UI.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2765">CDAP-2765</a> -
Fixed a problem with security warnings when CLI starts up.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2766">CDAP-2766</a> -
Fixed a problem with CLI asking for the user/password twice.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2767">CDAP-2767</a> -
Fixed a problem with incorrect error messages for namespace deletion.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2768">CDAP-2768</a> -
Fixed a problem with CLI and UI listing system.queue as a dataset.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2769">CDAP-2769</a> -
Fixed a problem with Use io.cdap.cdap.common.app.RunIds instead of
org.apache.twill.internal.RunIds for InMemoryServiceProgramRunner.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2787">CDAP-2787</a> -
Fixed a problem when the number of MapReduce task metrics going over limit and causing MapReduce to fail.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2796">CDAP-2796</a> -
Fixed a problem with emitting duplicate metrics for dataset ops.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2803">CDAP-2803</a> -
Fixed a problem with scan operations not reflecting in dataset ops metrics.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2804">CDAP-2804</a> -
Fixed a problem with DataSetRecordReader incorrectly reporting dataset ops metrics.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2810">CDAP-2810</a> -
Fixed a problem with IncrementAndGet, CompareAndSwap, and Delete ops on Table incorrectly
reporting two writes each.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2821">CDAP-2821</a> -
Fixed a problem with a Spark native library linkage error causing Standalone CDAP to stop.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2823">CDAP-2823</a> -
Fixed a problem with the conversion from Avro and to Avro not taking into account nested records.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2830">CDAP-2830</a> -
Fixed a problem with CDAP UI dying when CDAP Master is killed.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2832">CDAP-2832</a> -
Fixed a problem where suspending a schedule takes a long time and the CDAP UI does not provide any indication.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2838">CDAP-2838</a> -
Fixed a problem with poor error message when there is a mistake in security configration.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2839">CDAP-2839</a> -
Fixed a problem with the CDAP start script needing updating for the correct Node.js version.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2848">CDAP-2848</a> -
Fixed a problem with the Preferences Client test.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2849">CDAP-2849</a> -
Fixed a problem with the FileBatchSource reading files in twice if it takes longer that
one workflow cycle to complete the job.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2851">CDAP-2851</a> -
Fixed a problem with RPM and DEB release artifacts being uploaded to incorrect staging directory.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2854">CDAP-2854</a> -
Fixed a problem with the instructions for using Docker.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2855">CDAP-2855</a> -
Fixed a problem with the example builds in VM failing with a Maven dependency error.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2860">CDAP-2860</a> -
Fixed a problem with the documentation for updating dataset properties.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2861">CDAP-2861</a> -
Fixed a problem with CDAP UI not mentioning required fields in all entry forms.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2862">CDAP-2862</a> -
Fixed a problem with CDAP UI creating multiple namespaces with the same name.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2866">CDAP-2866</a> -
Fixed a problem with FileBatchSource not reattempting to read in files if there is a failure.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2870">CDAP-2870</a> -
Fixed a problem with Workflow Diagrams.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2871">CDAP-2871</a> -
Fixed a problem with the Cloudera Manager Hbase Gateway dependency.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2895">CDAP-2895</a> -
Fixed a problem with a put operation on the WorkflowToken not throwing an exception.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2899">CDAP-2899</a> -
Fixed a problem with Mapreduce local dirs not getting cleaned up.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2900">CDAP-2900</a> -
Fixed a problem with exposing app.template.dir as a config option.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2904">CDAP-2904</a> -
Fixed a problem with “Make Request” button overlapping with paths when a path is long.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2912">CDAP-2912</a> -
Fixed a problem with HBaseQueueDebugger not sorting queue barriers correctly.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2922">CDAP-2922</a> -
Fixed a problem with datasets created through DynamicDatasetContext not having metrics context.
Datasets in MapReduce and Spark programs, and workers, were not emitting metrics.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2925">CDAP-2925</a> -
Fixed a problem with the documentation on how to create datasets with properties.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2932">CDAP-2932</a> -
Fixed a problem with the AdapterClient getRuns method constructing a malformed URL.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2935">CDAP-2935</a> -
Fixed a problem with the logs endpoint to retrieve the latest entry not working correctly.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2940">CDAP-2940</a> -
Fixed a problem with the test case ArtifactStoreTest#testConcurrentSnapshotWrite.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2941">CDAP-2941</a> -
Fixed a problem with the ScriptTransform failing to initialize.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2942">CDAP-2942</a> -
Fixed a problem with the CDAP UI namespace dropdown failing on standalone restart.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2948">CDAP-2948</a> -
Fixed a problem with creating Adapters.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2952">CDAP-2952</a> -
Fixed a problem with the plugin avro library not being accessible to MapReduce.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2955">CDAP-2955</a> -
Fixed a problem with a NoSuchMethodException when trying to explore Datasets/Stream.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2971">CDAP-2971</a> -
Fixed a problem with the dataset registration not registering datasets for applications upon deploy.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2972">CDAP-2972</a> -
Fixed a problem with being unable to instantiate dataset in ETLWorker initialization.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2981">CDAP-2981</a> -
Fixed a problem with undoing a FileSets upgrade in favor of versioning and backward-compatibility.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2991">CDAP-2991</a> -
Fixed a problem with Explore not working when it launches a MapReduce job.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2992">CDAP-2992</a> -
Fixed a problem with CLI broken for secure CDAP.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2996">CDAP-2996</a> -
Fixed a problem with CDAP UI: Stop Run and Suspend Run buttons needed styling updates.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2997">CDAP-2997</a> -
Fixed a problem with SparkProgramRunnerTest failing randomly.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2999">CDAP-2999</a> -
Fixed a problem with MapReduce jobs showing the duration for tasks as 17 days before the mapper starts.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3001">CDAP-3001</a> -
Fixed a problem with truncating a custom dataset failing with internal server error.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3002">CDAP-3002</a> -
Fixed a problem with tick initialDelay not working properly.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3003">CDAP-3003</a> -
Fixed a problem with user metrics emitted from flowlets not being queryable using the flow’s tags.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3006">CDAP-3006</a> -
Fixed a problem with updating cdap-spark-* archetypes.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3007">CDAP-3007</a> -
Fixed a problem with testing all Spark apps/guides to work with 3.1 (in dist mode).</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3009">CDAP-3009</a> -
Fixed a problem with the stream conversion upgrade being in the upgrade tool in 3.1.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3010">CDAP-3010</a> -
Fixed a problem with a Bower Dependency Error.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3011">CDAP-3011</a> -
Fixed a problem with the IncrementSummingScannerTest failing intermittently.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3012">CDAP-3012</a> -
Fixed a problem with the DistributedWorkflowProgramRunner localizing the
spark-assembly.jar even if the workflow does not contain a Spark program.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3013">CDAP-3013</a> -
Fixed a problem with excluding a Spark assembly jar when building a MapReduce job jar.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3019">CDAP-3019</a> -
Fixed a problem with the PartitionedFileSet dropPartition not deleting files under the partition.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3021">CDAP-3021</a> -
Fixed a problem with allowing Cloudfront data to use BatchFileFilter.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3023">CDAP-3023</a> -
Fixed a problem with flowlet instance count defaulting to 1.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3024">CDAP-3024</a> -
Fixed a problem with surfacing more logs in CDAP UI for System Services.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3026">CDAP-3026</a> -
Fixed a problem with updating SparkPageRank example docs.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3027">CDAP-3027</a> -
Fixed a problem with the DFSStreamHeartbeatsTest failing on clusters.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3030">CDAP-3030</a> -
Fixed a problem with the loading of custom datasets being broken after upgrading.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3031">CDAP-3031</a> -
Fixed a problem with deploying an app with a dataset with an invalid base path returning an “internal error”.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3037">CDAP-3037</a> -
Fixed a problem with not being able to use a PartitionedFileSet in a custom dataset. If
a custom dataset embedded a Table and a PartitionedFileSet, loading the dataset at
runtime would fail.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3038">CDAP-3038</a> -
Fixed a problem with logs not showing up in UI when using Spark.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3039">CDAP-3039</a> -
Fixed a problem with worker not stopping at the end of a run method in standalone.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3040">CDAP-3040</a> -
Fixed a problem with flowlet and stream metrics not being available in distributed mode.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3042">CDAP-3042</a> -
Fixed a problem with the BufferingTable not merging buffered writes with multi-get results.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3043">CDAP-3043</a> -
Fixed a problem with the Javadocs being broken.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3044">CDAP-3044</a> -
Fixed a problem with the user service ‘methods’ field in service specifications being inaccurate.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3058">CDAP-3058</a> -
Fixed a problem with the NamespacedLocationFactory not appending correctly.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3066">CDAP-3066</a> -
Fixed a problem with FileBatchSource not failing properly.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3067">CDAP-3067</a> -
Fixed a problem with the UpgradeTool throwing a NullPointerException during UsageRegistry.upgrade().</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3070">CDAP-3070</a> -
Fixed a problem on Ubuntu 14.10 where removing JSON files from templates/plugins/ETLBatch breaks adapters.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3072">CDAP-3072</a> -
Fixed a problem with a documentation JavaScript bug.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3073">CDAP-3073</a> -
Fixed a problem with out-of-memory perm gen space.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3085">CDAP-3085</a> -
Fixed a problem with adding integration tests for datasets.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3086">CDAP-3086</a> -
Fixed a problem with the CDAP UI current adapter UI.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3087">CDAP-3087</a> -
Fixed a problem with CDAP UI: a session timeout on secure mode.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3088">CDAP-3088</a> -
Fixed a problem with CDAP UI: application types need to be updated.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3092">CDAP-3092</a> -
Fixed a problem with reading multiple files with one mapper in FileBatchSource.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3096">CDAP-3096</a> -
Fixed a problem with running MapReduce on HDP 2.2.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3098">CDAP-3098</a> -
Fixed problems with the CDAP UI Adapter UI.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3099">CDAP-3099</a> -
Fixed a problem with CDAP UI and that settings icons shift 2px when you click on them.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3104">CDAP-3104</a> -
Fixed a problem with CDAP Explore throwing an exception if a Table dataset does not set schema.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3105">CDAP-3105</a> -
Fixed a problem with LogParserTransform needing to emit HTTP status code info.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3106">CDAP-3106</a> -
Fixed a problem with Hive query - local MapReduce task failure on CDH-5.4.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3125">CDAP-3125</a> -
Fixed a problem with the WorkerProgramRunnerTest failing intermittently.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3127">CDAP-3127</a> -
Fixed a problem with the Kafka guide not working with CDAP 3.1.0.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3132">CDAP-3132</a> -
Fixed a problem with the ProgramLifecycleHttpHandlerTest failing intermittently.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3145">CDAP-3145</a> -
Fixed a problem with the Metrics processor not processing metrics.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3146">CDAP-3146</a> -
Fixed a problem with the CDAP VM build failing to instal the Eclipse plugin.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3148">CDAP-3148</a> -
Fixed a problem with CDAP Explore MapReduce queries failing due to MR-framework being
localized in the mapper container.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3149">CDAP-3149</a> -
Fixed a problem with cycles in an adapter create page causing the browser to freeze.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3151">CDAP-3151</a> -
Fixed a problem with CDAP examples shipped with SDK using JDK 1.6.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3161">CDAP-3161</a> -
Fixed a problem with MapReduce no longer working with default Cloudera manager settings.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3173">CDAP-3173</a> -
Fixed a problem with upgrading to 3.1.0 crashing the HBase co-processor.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3174">CDAP-3174</a> -
Fixed a problem with the ETL source/transform/sinks descriptions and documentation.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3175">CDAP-3175</a> -
Fixed a problem with the AbstractFlowlet constructors being deprecated when they should not be.</li>
</ul>
</div>
<div class="section" id="id114">
<h3>Deprecated and Removed Features<a class="headerlink" href="#id114" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li>See the <a class="reference external" href="http://docs.cask.co/cdap/3.1.0/en/reference-manual/javadocs/index.html#javadocs">CDAP 3.1.0 Javadocs</a>
for a list of deprecated and removed APIs.</li>
</ul>
</div>
<div class="section" id="known-issues-310">
<span id="id115"></span><h3>Known Issues<a class="headerlink" href="#known-issues-310" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li>See the above section (<em>API Changes</em>) for alterations that can affect existing installations.</li>
<li>CDAP has been tested on and supports CDH 4.2.x through CDH 5.4.4, HDP 2.0 through 2.6, and
Apache Bigtop 0.8.0. It has not been tested on more recent versions of CDH.
See <a class="reference external" href="http://docs.cask.co/cdap/3.1.0/en/admin-manual/installation/installation.html#install-hadoop-hbase">our Hadoop/HBase Environment configurations</a>.</li>
<li>After upgrading CDAP from a pre-3.0 version, any unprocessed metrics data in Kafka will
be lost and <em>WARN</em> log messages will be logged that tell about the inability to process
old data in the old format.</li>
<li>Retrieving multiple metrics—by issuing an HTTP POST request with a JSON list as
the request body that enumerates the name and attributes for each metric—is currently not
supported in the <a class="reference external" href="http://docs.cask.co/cdap/2.8.0/en/reference-manual/http-restful-api/http-restful-api-v3/metrics.html#query-tips">Metrics HTTP RESTful API v3</a>.
Instead, use the v2 API. It will be supported in a future release.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-797">CDAP-797</a> -
When running secure Hadoop clusters, metrics and debug logs from MapReduce programs are
not available.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-1007">CDAP-1007</a> -
If the Hive Metastore is restarted while the CDAP Explore Service is running, the
Explore Service remains alive, but becomes unusable. To correct, restart the CDAP Master—which will restart all services—as described under “Starting CDAP Services”
for your particular Hadoop distribution in the <a class="reference external" href="http://docs.cask.co/cdap/3.0.0/en/admin-manual/installation/installation.html#starting-services">Installation documentation</a>.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-1587">CDAP-1587</a> -
CDAP internally creates tables in the “user” space that begin with the word
<code class="docutils literal notranslate"><span class="pre">&quot;system&quot;</span></code>. User datasets with names starting with <code class="docutils literal notranslate"><span class="pre">&quot;system&quot;</span></code> can conflict if they
were to match one of those names. To avoid this, do not start any datasets with the word
<code class="docutils literal notranslate"><span class="pre">&quot;system&quot;</span></code>.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-1864">CDAP-1864</a> -
Creating a dataset in a non-existent namespace manifests in the RESTful API with an
incorrect error message.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2632">CDAP-2632</a> -
The application in the <a class="reference external" href="https://github.com/cdap-guides/cdap-kafka-ingest-guide/tree/release/cdap-3.0-compatible">cdap-kafka-ingest-guide</a>
does not run on Ubuntu 14.x as of CDAP 3.0.x.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2785">CDAP-2785</a> -
In the CDAP UI, many buttons will remain in focus after being clicked, even if they
should not retain focus.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2831">CDAP-2831</a> -
A workflow that is scheduled by time will not be run between the failure of the primary
master and the time that the secondary takes over. This scheduled run will not be
triggered at all.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2878">CDAP-2878</a> -
The semantics for TTL are confusing, in that the Table TTL property is
interpreted as milliseconds in some contexts: <code class="docutils literal notranslate"><span class="pre">DatasetDefinition.confgure()</span></code> and
<code class="docutils literal notranslate"><span class="pre">getAdmin()</span></code>.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-2945">CDAP-2945</a> -
If the input partition filter for a PartitionedFileSet does not match any partitions,
MapReduce jobs can fail.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3000">CDAP-3000</a> -
The Workflow token is in an inconsistent state for nodes in a fork while the nodes of
the fork are still running. It becomes consistent after the join.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3101">CDAP-3101</a> -
If there are more than 30 concurrent runs of a workflow, the runs will not be scheduled
due to a Quartz exception.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3179">CDAP-3179</a> -
If you are using CDH 5.3 (CDAP 3.0.0) and are upgrading to CDH 5.4 (CDAP 3.1.0), you
must first upgrade the underlying HBase before you upgrade CDAP. This means that you perform the
CDH upgrade before upgrading the CDAP.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3189">CDAP-3189</a> -
Large MapReduce jobs can cause excessive logging in the CDAP logs.</li>
<li><a class="reference external" href="https://issues.cask.co/browse/CDAP-3221">CDAP-3221</a> -
When running in Standalone CDAP, if a MapReduce job fails repeatedly, then the SDK
hits an out-of-memory exception due to <code class="docutils literal notranslate"><span class="pre">perm</span> <span class="pre">gen</span></code>. The Standalone needs restarting at
this point.</li>
</ul>
</div>
</div>
<div class="section" id="release-3-0-3">
<h2><a class="reference external" href="http://docs.cask.co/cdap/3.0.3/index.html">Release 3.0.3</a><a class="headerlink" href="#release-3-0-3" title="Permalink to this headline">🔗</a></h2>
<div class="section" id="id116">
<h3>Bug Fix<a class="headerlink" href="#id116" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li>Fixed a Bower dependency error in the CDAP UI
(<a class="reference external" href="https://issues.cask.co/browse/CDAP-3010">CDAP-3010</a>).</li>
</ul>
</div>
<div class="section" id="id117">
<h3>Known Issues<a class="headerlink" href="#id117" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li>See the <em>Known Issues</em> of <a class="reference external" href="#known-issues-301">version 3.0.1</a>.</li>
</ul>
</div>
</div>
<div class="section" id="release-3-0-2">
<h2><a class="reference external" href="http://docs.cask.co/cdap/3.0.2/index.html">Release 3.0.2</a><a class="headerlink" href="#release-3-0-2" title="Permalink to this headline">🔗</a></h2>
<div class="section" id="id118">
<h3>Bug Fixes<a class="headerlink" href="#id118" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li>Fixed problems with the dataset upgrade tool
(<a class="reference external" href="https://issues.cask.co/browse/CDAP-2962">CDAP-2962</a>,
<a class="reference external" href="https://issues.cask.co/browse/CDAP-2897">CDAP-2897</a>).</li>
</ul>
</div>
<div class="section" id="id119">
<h3>Known Issues<a class="headerlink" href="#id119" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li>See the <em>Known Issues</em> of <a class="reference external" href="#known-issues-301">version 3.0.1</a>.</li>
</ul>
</div>
</div>
<div class="section" id="release-3-0-1">
<h2><a class="reference external" href="http://docs.cask.co/cdap/3.0.1/index.html">Release 3.0.1</a><a class="headerlink" href="#release-3-0-1" title="Permalink to this headline">🔗</a></h2>
<div class="section" id="id121">
<h3>New Features<a class="headerlink" href="#id121" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li>In the CDAP UI, mandatory parameters for Application Template creation are marked with
asterisks, and if a user tries to create a template without one of those parameters, the
missing parameter is highlighted
(<a class="reference external" href="https://issues.cask.co/browse/CDAP-2499">CDAP-2499</a>).</li>
</ul>
</div>
<div class="section" id="id122">
<h3>Improvements<a class="headerlink" href="#id122" title="Permalink to this headline">🔗</a></h3>
<p><strong>Tools</strong></p>
<ul class="simple">
<li>Added a tool (<a class="reference external" href="https://github.com/cdapio/cdap/blob/release/3.0/cdap-master/src/main/java/io/cdap/cdap/data/tools/HBaseQueueDebugger.java">HBaseQueueDebugger</a>)
that counts consumed and unconsumed entries in a flowlet queue
(<a class="reference external" href="https://issues.cask.co/browse/CDAP-2105">CDAP-2105</a>).</li>
</ul>
<p><strong>CDAP UI</strong></p>
<ul class="simple">
<li>The currently executing node of a workflow is now highlighted in the CDAP UI
(<a class="reference external" href="https://issues.cask.co/browse/CDAP-2615">CDAP-2615</a>).</li>
<li>The list of datasets and the run histories in the CDAP UI are now paginated
(<a class="reference external" href="https://issues.cask.co/browse/CDAP-2626">CDAP-2626</a>,
<a class="reference external" href="https://issues.cask.co/browse/CDAP-2627">CDAP-2627</a>).</li>
<li>Added improvements to the CDAP UI when creating Application Templates
(<a class="reference external" href="https://issues.cask.co/browse/CDAP-2601">CDAP-2601</a>,
<a class="reference external" href="https://issues.cask.co/browse/CDAP-2602">CDAP-2602</a>,
<a class="reference external" href="https://issues.cask.co/browse/CDAP-2603">CDAP-2603</a>,
<a class="reference external" href="https://issues.cask.co/browse/CDAP-2605">CDAP-2605</a>,
<a class="reference external" href="https://issues.cask.co/browse/CDAP-2606">CDAP-2606</a>,
<a class="reference external" href="https://issues.cask.co/browse/CDAP-2607">CDAP-2607</a>,
<a class="reference external" href="https://issues.cask.co/browse/CDAP-2610">CDAP-2610</a>).</li>
<li>Improved the error messages returned when there are problems creating Application
Templates in the CDAP UI
(<a class="reference external" href="https://issues.cask.co/browse/CDAP-2597">CDAP-2597</a>).</li>
</ul>
<p><strong>CDAP SDK VM</strong></p>
<ul class="simple">
<li>Added the Apache Flume agent flume-ng to the CDAP SDK VM
(<a class="reference external" href="https://issues.cask.co/browse/CDAP-2612">CDAP-2612</a>).</li>
<li>Added the ability to copy and paste to the CDAP SDK VM
(<a class="reference external" href="https://issues.cask.co/browse/CDAP-2611">CDAP-2611</a>).</li>
<li>Pre-downloaded the example dependencies into the CDAP SDK VM to speed building of the
CDAP examples
(<a class="reference external" href="https://issues.cask.co/browse/CDAP-2613">CDAP-2613</a>).</li>
</ul>
</div>
<div class="section" id="id123">
<h3>Bug Fixes<a class="headerlink" href="#id123" title="Permalink to this headline">🔗</a></h3>
<p><strong>General</strong></p>
<ul class="simple">
<li>Fixed a problem with the HBase store and flows with multiple queues, where one queue name
is a prefix of another queue name
(<a class="reference external" href="https://issues.cask.co/browse/CDAP-1996">CDAP-1996</a>).</li>
<li>Fixed a problem with namespaces with underscores in the name crashing the Hadoop HBase
region servers
(<a class="reference external" href="https://issues.cask.co/browse/CDAP-2110">CDAP-2110</a>).</li>
<li>Removed the requirement to specify the JDBC driver class property twice in the adaptor
configuration for Database Sources and Sinks
(<a class="reference external" href="https://issues.cask.co/browse/CDAP-2453">CDAP-2453</a>).</li>
<li>Fixed a problem in Distributed CDAP where the status of running program always returns
as “STOPPED” when the CDAP Master is restarted
(<a class="reference external" href="https://issues.cask.co/browse/CDAP-2489">CDAP-2489</a>).</li>
<li>Fixed a problem with invalid RunRecords for Spark and MapReduce programs that are run as
part of a Workflow (<a class="reference external" href="https://issues.cask.co/browse/CDAP-2490">CDAP-2490</a>).</li>
<li>Fixed a problem with the CDAP Master not being HA (highly available) when a leadership
change happens
(<a class="reference external" href="https://issues.cask.co/browse/CDAP-2495">CDAP-2495</a>).</li>
<li>Fixed a problem with upgrading of queues with the UpgradeTool
(<a class="reference external" href="https://issues.cask.co/browse/CDAP-2502">CDAP-2502</a>).</li>
<li>Fixed a problem with ObjectMappedTables not deleting missing fields when updating a row
(<a class="reference external" href="https://issues.cask.co/browse/CDAP-2523">CDAP-2523</a>,
<a class="reference external" href="https://issues.cask.co/browse/CDAP-2524">CDAP-2524</a>).</li>
<li>Fixed a problem with a stream not being created properly when deploying an application
after the default namespace was deleted
(<a class="reference external" href="https://issues.cask.co/browse/CDAP-2537">CDAP-2537</a>).</li>
<li>Fixed a problem with the Applicaton Template Kafka Source not using the persisted offset
when the Adapter is restarted
(<a class="reference external" href="https://issues.cask.co/browse/CDAP-2547">CDAP-2547</a>).</li>
<li>A problem with CDAP using its own transaction snapshot codec, leading to huge snapshot
files and OutOfMemory exceptions, and transaction snapshots that can’t be read using
Tephra’s tools, has been resolved by replacing the codec with Tephra’s SnapshotCodecV3
(<a class="reference external" href="https://issues.cask.co/browse/CDAP-2563">CDAP-2563</a>,
<a class="reference external" href="https://issues.cask.co/browse/CDAP-2946">CDAP-2946</a>,
<a class="reference external" href="https://issues.cask.co/browse/TEPHRA-101">TEPHRA-101</a>).</li>
<li>Fixed a problem with CDAP Master not being resilient in the handling of ZooKeeper
exceptions
(<a class="reference external" href="https://issues.cask.co/browse/CDAP-2569">CDAP-2569</a>).</li>
<li>Fixed a problem with RunRecords not being cleaned up correctly after certain exceptions
(<a class="reference external" href="https://issues.cask.co/browse/CDAP-2584">CDAP-2584</a>).</li>
<li>Fixed a problem with the CDAP Maven archetype having an incorrect CDAP version in it
(<a class="reference external" href="https://issues.cask.co/browse/CDAP-2634">CDAP-2634</a>).</li>
<li>Fixed a problem with the description of the TwitterSource not describing its output
(<a class="reference external" href="https://issues.cask.co/browse/CDAP-2648">CDAP-2648</a>).</li>
<li>Fixed a problem with the Twitter Source not handling missing fields correctly and as a
consequence producing tweets (with errors) that were then not stored on disk
(<a class="reference external" href="https://issues.cask.co/browse/CDAP-2653">CDAP-2653</a>).</li>
<li>Fixed a problem with the TwitterSource not calculating the time of tweet correctly
(<a class="reference external" href="https://issues.cask.co/browse/CDAP-2656">CDAP-2656</a>).</li>
<li>Fixed a problem with the JMS Real-time Source failing to load required plugin sources
(<a class="reference external" href="https://issues.cask.co/browse/CDAP-2661">CDAP-2661</a>).</li>
<li>Fixed a problem with executing Hive queries on a distributed CDAP due to a failure to
load Grok classes
(<a class="reference external" href="https://issues.cask.co/browse/CDAP-2678">CDAP-2678</a>).</li>
<li>Fixed a problem with CDAP Program jars not being cleaned up from the temporary directory
(<a class="reference external" href="https://issues.cask.co/browse/CDAP-2698">CDAP-2698</a>).</li>
<li>Fixed a problem with ProjectionTransforms not handling input data fields with null
values correctly
(<a class="reference external" href="https://issues.cask.co/browse/CDAP-2719">CDAP-2719</a>).</li>
<li>Fixed a problem with the CDAP SDK running out of memory when MapReduce jobs are run repeatedly
(<a class="reference external" href="https://issues.cask.co/browse/CDAP-2743">CDAP-2743</a>).</li>
<li>Fixed a problem with not using CDAP RunIDs in the in-memory version of the CDAP SDK
(<a class="reference external" href="https://issues.cask.co/browse/CDAP-2769">CDAP-2769</a>).</li>
</ul>
<p><strong>CDAP CLI</strong></p>
<ul class="simple">
<li>Fixed a problem with the CDAP CLI not printing an error if it is unable to connect to a
CDAP instance
(<a class="reference external" href="https://issues.cask.co/browse/CDAP-2529">CDAP-2529</a>).</li>
<li>Fixed a problem with extra whitespace in commands entered into the CDAP CLI causing errors
(<a class="reference external" href="https://issues.cask.co/browse/CDAP-2538">CDAP-2538</a>).</li>
</ul>
<p><strong>CDAP SDK Standalone</strong></p>
<ul class="simple">
<li>Updated the messages displayed when starting the Standalone CDAP SDK as to components
and the JVM required (<a class="reference external" href="https://issues.cask.co/browse/CDAP-2445">CDAP-2445</a>).</li>
<li>Fixed a problem with the creation of the default namespace upon starting the CDAP SDK
(<a class="reference external" href="https://issues.cask.co/browse/CDAP-2587">CDAP-2587</a>).</li>
</ul>
<p><strong>CDAP SDK VM</strong></p>
<ul class="simple">
<li>Fixed a problem with using the default namespace on the CDAP SDK Virtual Machine Image
(<a class="reference external" href="https://issues.cask.co/browse/CDAP-2500">CDAP-2500</a>).</li>
<li>Fixed a problem with the VirtualBox VM retaining a MAC address obtained from the build host
(<a class="reference external" href="https://issues.cask.co/browse/CDAP-2640">CDAP-2640</a>).</li>
</ul>
<p><strong>CDAP UI</strong></p>
<ul class="simple">
<li>Fixed a problem with incorrect flow metrics showing in the CDAP UI
(<a class="reference external" href="https://issues.cask.co/browse/CDAP-2494">CDAP-2494</a>).</li>
<li>Fixed a problem in the CDAP UI with the properties in the Projection Transform being
displayed inconsistently
(<a class="reference external" href="https://issues.cask.co/browse/CDAP-2525">CDAP-2525</a>).</li>
<li>Fixed a problem in the CDAP UI not automatically updating the number of flowlet instances
(<a class="reference external" href="https://issues.cask.co/browse/CDAP-2534">CDAP-2534</a>).</li>
<li>Fixed a problem in the CDAP UI with a window resize preventing clicking of the Adapter
Template drop down menu
(<a class="reference external" href="https://issues.cask.co/browse/CDAP-2573">CDAP-2573</a>).</li>
<li>Fixed a problem with the CDAP UI not performing validation of mandatory parameters
before the creation of an adapter
(<a class="reference external" href="https://issues.cask.co/browse/CDAP-2575">CDAP-2575</a>).</li>
<li>Fixed a problem with an incorrect version of CDAP being shown in the CDAP UI
(<a class="reference external" href="https://issues.cask.co/browse/CDAP-2586">CDAP-2586</a>).</li>
<li>Reduced the number of clicks required to navigate and perform actions within the CDAP UI
(<a class="reference external" href="https://issues.cask.co/browse/CDAP-2622">CDAP-2622</a>,
<a class="reference external" href="https://issues.cask.co/browse/CDAP-2625">CDAP-2625</a>).</li>
<li>Fixed a problem with an additional forward-slash character in the URL causing a “page
not found error” in the CDAP UI
(<a class="reference external" href="https://issues.cask.co/browse/CDAP-2624">CDAP-2624</a>).</li>
<li>Fixed a problem with the error dropdown of the CDAP UI not scrolling when it has a
large number of errors
(<a class="reference external" href="https://issues.cask.co/browse/CDAP-2633">CDAP-2633</a>).</li>
<li>Fixed a problem in the CDAP UI with the Twitter Source’s consumer key secret not being
treated as a password field
(<a class="reference external" href="https://issues.cask.co/browse/CDAP-2649">CDAP-2649</a>).</li>
<li>Fixed a problem with the CDAP UI attempting to create an adapter without a name
(<a class="reference external" href="https://issues.cask.co/browse/CDAP-2652">CDAP-2652</a>).</li>
<li>Fixed a problem with the CDAP UI not being able to find the ETL plugin templates on
distributed CDAP
(<a class="reference external" href="https://issues.cask.co/browse/CDAP-2655">CDAP-2655</a>).</li>
<li>Fixed a problem with the CDAP UI’s System Dashboard chart having a y-axis starting at “-200”
(<a class="reference external" href="https://issues.cask.co/browse/CDAP-2699">CDAP-2699</a>).</li>
<li>Fixed a problem with the rendering of stack trace logs in the CDAP UI
(<a class="reference external" href="https://issues.cask.co/browse/CDAP-2745">CDAP-2745</a>).</li>
<li>Fixed a problem with the CDAP UI not working with secure CDAP instances, either clusters
or standalone
(<a class="reference external" href="https://issues.cask.co/browse/CDAP-2770">CDAP-2770</a>).</li>
<li>Fixed a problem with the coloring of completed runs of Workflow DAGs in the CDAP UI
(<a class="reference external" href="https://issues.cask.co/browse/CDAP-2781">CDAP-2781</a>).</li>
</ul>
<p><strong>Documentation</strong></p>
<ul class="simple">
<li>Fixed errors with the documentation examples of the ETL Plugins
(<a class="reference external" href="https://issues.cask.co/browse/CDAP-2503">CDAP-2503</a>).</li>
<li>Documented the licenses of all shipped CDAP UI components
(<a class="reference external" href="https://issues.cask.co/browse/CDAP-2582">CDAP-2582</a>).</li>
<li>Corrected issues with the building of Javadocs used on the website and removed Javadocs
previously included in the SDK
(<a class="reference external" href="https://issues.cask.co/browse/CDAP-2730">CDAP-2730</a>).</li>
<li>Added a recommended version (v.12.0) of Node.js to the documentation
(<a class="reference external" href="https://issues.cask.co/browse/CDAP-2762">CDAP-2762</a>).</li>
</ul>
</div>
<div class="section" id="known-issues-301">
<span id="id124"></span><h3>Known Issues<a class="headerlink" href="#known-issues-301" title="Permalink to this headline">🔗</a></h3>
<ul>
<li><p class="first">The application in the <a class="reference external" href="https://github.com/cdap-guides/cdap-kafka-ingest-guide/tree/release/cdap-3.0-compatible">cdap-kafka-ingest-guide</a>
does not run on Ubuntu 14.x and CDAP 3.0.x
(<a class="reference external" href="https://issues.cask.co/browse/CDAP-2632">CDAP-2632</a>,
<a class="reference external" href="https://issues.cask.co/browse/CDAP-2749">CDAP-2749</a>).</p>
</li>
<li><p class="first">Metrics for <a class="reference external" href="http://docs.cask.co/cdap/3.0.1/en/developers-manual/building-blocks/datasets/fileset.html#timepartitionedfileset">TimePartitionedFileSets</a>
can show zero values even if there is data present
(<a class="reference external" href="https://issues.cask.co/browse/CDAP-2721">CDAP-2721</a>).</p>
</li>
<li><p class="first">In the CDAP UI: many buttons will remain in focus after being clicked, even if they
should not retain focus
(<a class="reference external" href="https://issues.cask.co/browse/CDAP-2785">CDAP-2785</a>).</p>
</li>
<li><p class="first">When the CDAP-Master dies, the CDAP UI does not repsond appropriately, and instead of
waiting for routing to the secondary master to begin, it loses its connection
(<a class="reference external" href="https://issues.cask.co/browse/CDAP-2830">CDAP-2830</a>).</p>
</li>
<li><p class="first">A workflow that is scheduled by time will not be run between the failure of the primary
master and the time that the secondary takes over. This scheduled run will not be
triggered at all. There is no warnings or messages about the missed run of the
workflow. (<a class="reference external" href="https://issues.cask.co/browse/CDAP-2831">CDAP-2831</a>)</p>
</li>
<li><p class="first">CDAP has been tested on and supports CDH 4.2.x through CDH 5.3.x, HDP 2.0 through 2.1, and
Apache Bigtop 0.8.0. It has not been tested on more recent versions of CDH.
See <a class="reference external" href="http://docs.cask.co/cdap/3.0.0/en/admin-manual/installation/installation.html#install-hadoop-hbase">our Hadoop/HBase Environment configurations</a>.</p>
</li>
<li><p class="first">After upgrading CDAP from a pre-3.0 version, any unprocessed metrics data in Kafka will
be lost and <em>WARN</em> log messages will be logged that tell about the inability to process
old data in the old format.</p>
</li>
<li><p class="first">See the above section (<em>API Changes</em>) for alterations that can affect existing installations.</p>
</li>
<li><p class="first">When running secure Hadoop clusters, metrics and debug logs from MapReduce programs are
not available (<a class="reference external" href="https://issues.cask.co/browse/CDAP-797">CDAP-797</a>).</p>
</li>
<li><p class="first">If the Hive Metastore is restarted while the CDAP Explore Service is running, the
Explore Service remains alive, but becomes unusable. To correct, <a class="reference external" href="http://docs.cask.co/cdap/3.0.0/en/admin-manual/installation/installation.html#starting-services">restart the CDAP Master</a>,
which will restart all services (<a class="reference external" href="https://issues.cask.co/browse/CDAP-1007">CDAP-1007</a>).</p>
</li>
<li><p class="first">User datasets with names starting with <code class="docutils literal notranslate"><span class="pre">&quot;system&quot;</span></code> can potentially cause conflicts
(<a class="reference external" href="https://issues.cask.co/browse/CDAP-1587">CDAP-1587</a>).</p>
</li>
<li><p class="first">Scaling the number of metrics processor instances doesn’t automatically distribute the
processing load to the newer instances of the metrics processor. The CDAP Master needs to be
restarted to effectively distribute the processing across all metrics processor instances
(<a class="reference external" href="https://issues.cask.co/browse/CDAP-1853">CDAP-1853</a>).</p>
</li>
<li><p class="first">Creating a dataset in a non-existent namespace manifests in the RESTful API with an
incorrect error message (<a class="reference external" href="https://issues.cask.co/browse/CDAP-1864">CDAP-1864</a>).</p>
</li>
<li><p class="first">Retrieving multiple metrics—by issuing an HTTP POST request with a JSON list as
the request body that enumerates the name and attributes for each metric—is currently not
supported in the
<a class="reference external" href="http://docs.cask.co/cdap/2.8.0/en/reference-manual/http-restful-api/http-restful-api-v3/metrics.html#query-tips">Metrics HTTP RESTful API v3</a>.
Instead, use the v2 API. It will be supported in a future release.</p>
</li>
<li><p class="first">Typically, datasets are bundled as part of applications. When an application is upgraded and redeployed,
any changes in datasets will not be redeployed. This is because datasets can be shared across applications,
and an incompatible schema change can break other applications that are using the dataset.
A workaround (<a class="reference external" href="https://issues.cask.co/browse/CDAP-1253">CDAP-1253</a>) is to allow <em>unchecked dataset upgrades</em>.
Upgrades cause the dataset metadata, i.e. its specification including properties, to be updated. The dataset
runtime code is also updated. To prevent data loss the existing data and the underlying HBase tables remain as-is.</p>
<p>You can allow <em>unchecked dataset upgrades</em> by setting the configuration property <code class="docutils literal notranslate"><span class="pre">dataset.unchecked.upgrade</span></code>
to <code class="docutils literal notranslate"><span class="pre">true</span></code> in <code class="docutils literal notranslate"><span class="pre">cdap-site.xml</span></code>. This will ensure that datasets are upgraded when the application is redeployed.
When this configuration is set, the recommended process to deploy an upgraded dataset is to first stop
all applications that are using the dataset before deploying the new version of the application.
This lets all containers (flows, services, etc) to pick up the new dataset changes.
When datasets are upgraded using <code class="docutils literal notranslate"><span class="pre">dataset.unchecked.upgrade</span></code>, no schema compatibility checks are performed by the
system. Hence it is very important that the developer verify the backward-compatibility, and makes sure that
other applications that are using the dataset can work with the new changes.</p>
</li>
</ul>
</div>
</div>
<div class="section" id="release-3-0-0">
<h2><a class="reference external" href="http://docs.cask.co/cdap/3.0.0/index.html">Release 3.0.0</a><a class="headerlink" href="#release-3-0-0" title="Permalink to this headline">🔗</a></h2>
<div class="section" id="id125">
<h3>New Features<a class="headerlink" href="#id125" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li>Support for application templates has been added (<a class="reference external" href="https://issues.cask.co/browse/CDAP-1753">CDAP-1753</a>).</li>
<li>Built-in ETL application templates and plugins have been added (<a class="reference external" href="https://issues.cask.co/browse/CDAP-1767">CDAP-1767</a>).</li>
<li>New <a class="reference external" href="http://docs.cask.co/cdap/3.0.0/en/admin-manual/operations/cdap-ui.html#cdap-ui">CDAP UI</a>,
supports creating ETL applications directly in the web UI.
See section below (<a class="reference external" href="#new-user-interface-300">New 3.0.0 User Interface</a>) for details.</li>
<li>Workflow logs can now be retrieved using the <a class="reference external" href="http://docs.cask.co/cdap/3.0.0/en/reference-manual/http-restful-api/logging.html#http-restful-api-logging">CDP HTTP Logging RESTful API</a>
(<a class="reference external" href="https://issues.cask.co/browse/CDAP-1089">CDAP-1089</a>).</li>
<li>Support has been added for suspending and resuming of a workflow (<a class="reference external" href="https://issues.cask.co/browse/CDAP-1610">CDAP-1610</a>).</li>
<li>Condition nodes in a workflow now allow branching based on a boolean predicate
(<a class="reference external" href="https://issues.cask.co/browse/CDAP-1928">CDAP-1928</a>).</li>
<li>Condition nodes in a workflow now allow passing the Hadoop counters from a MapReduce
program to following Condition nodes in the workflow (<a class="reference external" href="https://issues.cask.co/browse/CDAP-1611">CDAP-1611</a>).</li>
<li>Logs can now be fetched based on the <code class="docutils literal notranslate"><span class="pre">run-id</span></code> (<a class="reference external" href="https://issues.cask.co/browse/CDAP-1582">CDAP-1582</a>).</li>
<li>CDAP Tables are <a class="reference external" href="http://docs.cask.co/cdap/3.0.0/en/developers-manual/data-exploration/tables.html#table-exploration">now explorable</a>
(<a class="reference external" href="https://issues.cask.co/browse/CDAP-946">CDAP-946</a>).</li>
<li>The <a class="reference external" href="http://docs.cask.co/cdap/3.0.0/en/reference-manual/cli-api.html#cli">CDAP CLI</a>
supports the new <a class="reference external" href="http://docs.cask.co/cdap/3.0.0/en/application-templates/index.html">application template and adapters APIs</a>.
(<a class="reference external" href="https://issues.cask.co/browse/CDAP-1773">CDAP-1773</a>).</li>
<li>The <a class="reference external" href="http://docs.cask.co/cdap/3.0.0/en/reference-manual/cli-api.html#cli">CDAP CLI</a>
startup options have been changed to accommodate a new option
of executing a file containing a series of CLI commands, line-by-line.</li>
<li>Both <a class="reference external" href="http://logstash.net/docs/1.4.2/filters/grok">grok</a> and
<a class="reference external" href="http://en.wikipedia.org/wiki/Syslog">syslog</a> record formats can now be used when
<a class="reference external" href="http://docs.cask.co/cdap/3.0.0/en/reference-manual/http-restful-api/stream.html#http-restful-api-stream-setting-properties">setting the format of a stream</a>
(<a class="reference external" href="https://issues.cask.co/browse/CDAP-1949">CDAP-1949</a>).</li>
<li>Added HTTP RESTful endpoints for listing datasets and streams as used by adapters,
programs, and applications, and vice-versa
(<a class="reference external" href="https://issues.cask.co/browse/CDAP-2214">CDAP-2214</a>).</li>
<li>Created a <a class="reference external" href="https://github.com/cdapio/cdap/pull/2290">queue introspection tool</a>,
for counting processed and unprocessed entries in a
flowlet queue (<a class="reference external" href="https://issues.cask.co/browse/CDAP-2105">CDAP-2105</a>).</li>
<li>Support for CDAP SDK VM build automation has been added (<a class="reference external" href="https://issues.cask.co/browse/CDAP-2030">CDAP-2030</a>).</li>
<li>A Cube dataset has been added (<a class="reference external" href="https://issues.cask.co/browse/CDAP-1520">CDAP-1520</a>).</li>
<li>A Batch and Real-Time Cube dataset sink has been added (<a class="reference external" href="https://issues.cask.co/browse/CDAP-1966">CDAP-1520</a>).</li>
<li>Metrics and status information for MapReduce on a task level is now exposed (<a class="reference external" href="https://issues.cask.co/browse/CDAP-1958">CDAP-1520</a>).</li>
</ul>
</div>
<div class="section" id="new-user-interface">
<span id="new-user-interface-300"></span><h3>New User Interface<a class="headerlink" href="#new-user-interface" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li>Introduced a new UI, organization based on namespaces and users.</li>
<li>Users can switch between namespaces.</li>
<li>Uses web sockets to retrieve updates from the backend.</li>
<li><strong>Development Section</strong><ul>
<li>Introduces a UI for programs based on run-ids.</li>
<li>Users can view logs and, in certain cases—flows—flowlets, of a program based on run ids.</li>
<li>Shows a list of datasets and streams used by a program, and shows programs using a specific dataset and stream.</li>
<li>Shows the history of a program (list of runs).</li>
<li>Datasets or streams are explorable on a dataset/stream level or on a global level.</li>
<li>Shows program level metrics on under each program.</li>
</ul>
</li>
<li><strong>Operations section</strong><ul>
<li>Introduces an operations section to explore metrics.</li>
<li>Allows users to create custom dashboard with custom metrics.</li>
<li>Users can add different types of charts (line, bar, area, pie, donut, scatter, spline,
area-spline, area-spline-stacked, area-stacked, step, table).</li>
<li>Users can add multiple metrics on a single dashboard, or on a single widget on a single dashboard.</li>
<li>Users can organize the widgets in either a two, three, or four-column layout.</li>
<li>Users can toggle the frequency at which data is polled for a metric.</li>
<li>Users can toggle the resolution of data displayed in a graph.</li>
</ul>
</li>
<li><strong>Admin Section</strong><ul>
<li>Users can manage different objects of CDAP (applications, programs, datasets, and streams).</li>
<li>Users can create namespaces.</li>
<li>Through the Admin view, users can configure their preferences at the CDAP level, namespace level, or application level.</li>
<li>Users can manage the system services, applications, and streams through the Admin view.</li>
</ul>
</li>
<li><strong>Adapters</strong><ul>
<li>Users can create ETLBatch and ETLRealtime adapters from within the UI.</li>
<li>Users can choose from a list of plugins that comes by default with CDAP when creating an adapter.</li>
<li>Users can save an adapter as a draft, to be created at a later point-in-time.</li>
<li>Users can configure plugin properties with appropriate editors from within the UI when creating an adapter.</li>
</ul>
</li>
<li>The Old CDAP Console has been deprecated.</li>
</ul>
</div>
<div class="section" id="id126">
<h3>Improvement<a class="headerlink" href="#id126" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li>The <a class="reference external" href="http://docs.cask.co/cdap/3.0.0/en/reference-manual/http-restful-api/metrics.html#http-restful-api-metrics">metrics system APIs</a>
have been revised and improved
(<a class="reference external" href="https://issues.cask.co/browse/CDAP-1596">CDAP-1596</a>).</li>
<li>The metrics system performance has been improved
(<a class="reference external" href="https://issues.cask.co/browse/CDAP-2124">CDAP-2124</a>,
<a class="reference external" href="https://issues.cask.co/browse/CDAP-2125">CDAP-2125</a>).</li>
</ul>
</div>
<div class="section" id="id127">
<h3>Bug Fixes<a class="headerlink" href="#id127" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li>The CDAP Authentication server now reports the port correctly when the port is set to 0
(<a class="reference external" href="https://issues.cask.co/browse/CDAP-614">CDAP-614</a>).</li>
<li>History of the programs running under workflow (Spark and MapReduce) is now updated correctly
(<a class="reference external" href="https://issues.cask.co/browse/CDAP-1293">CDAP-1293</a>).</li>
<li>Programs running under a workflow now receive a unique <code class="docutils literal notranslate"><span class="pre">run-id</span></code>
(<a class="reference external" href="https://issues.cask.co/browse/CDAP-2025">CDAP-2025</a>).</li>
<li>RunRecords are now updated with the RuntimeService to account for node failures
(<a class="reference external" href="https://issues.cask.co/browse/CDAP-2202">CDAP-2202</a>).</li>
<li>MapReduce metrics are now available on a secure cluster
(<a class="reference external" href="https://issues.cask.co/browse/CDAP-64">CDAP-64</a>).</li>
</ul>
</div>
<div class="section" id="id128">
<h3>API Changes<a class="headerlink" href="#id128" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li>The endpoint (<code class="docutils literal notranslate"><span class="pre">POST</span> <span class="pre">'&lt;base-url&gt;/metrics/search?target=childContext[&amp;context=&lt;context&gt;]'</span></code>)
that searched for the available contexts of metrics has been deprecated, pending removal
in a later version of CDAP (<a class="reference external" href="https://issues.cask.co/browse/CDAP-1998">CDAP-1998</a>). A
<a class="reference external" href="http://docs.cask.co/cdap/3.0.0/en/reference-manual/http-restful-api/metrics.html#http-restful-api-metrics-search-for-contexts">replacement endpoint</a>
is available.</li>
<li>The endpoint (<code class="docutils literal notranslate"><span class="pre">POST</span> <span class="pre">'&lt;base-url&gt;/metrics/search?target=metric&amp;context=&lt;context&gt;'</span></code>)
that searched for metrics in a specified context has been deprecated, pending removal
in a later version of CDAP (<a class="reference external" href="https://issues.cask.co/browse/CDAP-1998">CDAP-1998</a>). A
<a class="reference external" href="http://docs.cask.co/cdap/3.0.0/en/reference-manual/http-restful-api/metrics.html#http-restful-api-metrics-search-for-metrics">replacement endpoint</a>
is available.</li>
<li>The endpoint (<code class="docutils literal notranslate"><span class="pre">POST</span> <span class="pre">'&lt;base-url&gt;/metrics/query?context=&lt;context&gt;[&amp;groupBy=&lt;tags&gt;]&amp;metric=&lt;metric&gt;&amp;&lt;time-range&gt;'</span></code>)
that queried for a metric has been deprecated, pending removal
in a later version of CDAP (<a class="reference external" href="https://issues.cask.co/browse/CDAP-1998">CDAP-1998</a>). A
<a class="reference external" href="http://docs.cask.co/cdap/3.0.0/en/reference-manual/http-restful-api/metrics.html#http-restful-api-metrics-querying-a-metric">replacement endpoint</a>
is available.</li>
<li>Metrics: The tag name for service handlers in previous releases was wrongly <code class="docutils literal notranslate"><span class="pre">&quot;runnable&quot;</span></code>,
and internally represented as <code class="docutils literal notranslate"><span class="pre">&quot;srn&quot;</span></code>. These metrics are now tagged as <code class="docutils literal notranslate"><span class="pre">&quot;handler&quot;</span></code> (<code class="docutils literal notranslate"><span class="pre">&quot;hnd&quot;</span></code>), and
metrics queries will only account for this tag name. If you need to query historic metrics
that were emitted with the old tag <code class="docutils literal notranslate"><span class="pre">&quot;runnable&quot;</span></code>, use <code class="docutils literal notranslate"><span class="pre">&quot;srn&quot;</span></code> to query them (instead of either
<code class="docutils literal notranslate"><span class="pre">&quot;runnable&quot;</span></code> or <code class="docutils literal notranslate"><span class="pre">&quot;handler&quot;</span></code>).</li>
<li>The <a class="reference external" href="http://docs.cask.co/cdap/3.0.0/en/reference-manual/cli-api.html#cli">CDAP CLI</a>
startup options have been changed to accommodate a new option
of executing a file containing a series of CLI commands, line-by-line.</li>
<li>The metrics system APIs have been improved (<a class="reference external" href="https://issues.cask.co/browse/CDAP-1596">CDAP-1596</a>).</li>
<li>The rules for <a class="reference external" href="http://docs.cask.co/cdap/3.0.0/en/reference-manual/http-restful-api/metrics.html#http-restful-api-metrics-time-range">resolving resolution</a>
when using <code class="docutils literal notranslate"><span class="pre">resolution=auto</span></code> in metrics query have been changed
(<a class="reference external" href="https://issues.cask.co/browse/CDAP-1922">CDAP-1922</a>).</li>
<li>Backward incompatible changes in <code class="docutils literal notranslate"><span class="pre">InputFormatProvider</span></code> and <code class="docutils literal notranslate"><span class="pre">OutputFormatProvider</span></code>.
It won’t affect user code that uses <code class="docutils literal notranslate"><span class="pre">FileSet</span></code> or <code class="docutils literal notranslate"><span class="pre">PartitionedFileSet</span></code>.
It only affects classes who implement the <code class="docutils literal notranslate"><span class="pre">InputFormatProvider</span></code> or <code class="docutils literal notranslate"><span class="pre">OutputFormatProvider</span></code>:<ul>
<li><code class="docutils literal notranslate"><span class="pre">InputFormatProvider.getInputFormatClass()</span></code> is removed and<ul>
<li>replaced with <code class="docutils literal notranslate"><span class="pre">InputFormatProvider.getInputFormatClassName()</span></code>;</li>
</ul>
</li>
<li><code class="docutils literal notranslate"><span class="pre">OutputFormatProvider.getOutputFormatClass()</span></code> is removed and<ul>
<li>replaced with <code class="docutils literal notranslate"><span class="pre">OutputFormatProvider.getOutputFormatClassName()</span></code>.</li>
</ul>
</li>
</ul>
</li>
</ul>
</div>
<div class="section" id="id129">
<h3>Deprecated and Removed Features<a class="headerlink" href="#id129" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li>The <a class="reference external" href="http://docs.cask.co/cdap/2.8.0/en/developers-manual/ingesting-tools/cdap-file-drop-zone.html">File DropZone</a>
and <a class="reference external" href="http://docs.cask.co/cdap/2.8.0/en/developers-manual/ingesting-tools/cdap-file-tailer.html">File Tailer</a>
are no longer supported as of Release 3.0.</li>
<li>Support for <em>procedures</em> has been removed. After upgrading, an application that
contained a procedure must be redeployed.</li>
<li>Support for <em>service workers</em> have been removed. After upgrading, an application that
contained a service worker must be redeployed.</li>
<li>The old CDAP Console has been deprecated.</li>
<li>Support for JDK/JRE 1.6 (Java 6) has ended; JDK/JRE 1.7 (Java 7) is
<a class="reference external" href="http://docs.cask.co/cdap/3.0.0/en/admin-manual/installation/installation.html#install-java-runtime">now required for CDAP</a> or the
<a class="reference external" href="http://docs.cask.co/cdap/3.0.0/en/developers-manual/getting-started/standalone/index.html#standalone-index">CDAP SDK</a>.</li>
</ul>
</div>
<div class="section" id="known-issues-300">
<span id="id130"></span><h3>Known Issues<a class="headerlink" href="#known-issues-300" title="Permalink to this headline">🔗</a></h3>
<ul>
<li><p class="first">CDAP has been tested on and supports CDH 4.2.x through CDH 5.3.x, HDP 2.0 through 2.1, and
Apache Bigtop 0.8.0. It has not been tested on more recent versions of CDH.
See <a class="reference external" href="http://docs.cask.co/cdap/3.0.0/en/admin-manual/installation/installation.html#install-hadoop-hbase">our Hadoop/HBase Environment configurations</a>.</p>
</li>
<li><p class="first">After upgrading CDAP from a pre-3.0 version, any unprocessed metrics data in Kafka will
be lost and <em>WARN</em> log messages will be logged that tell about the inability to process
old data in the old format.</p>
</li>
<li><p class="first">See the above section (<em>API Changes</em>) for alterations that can affect existing installations.</p>
</li>
<li><p class="first">When running secure Hadoop clusters, metrics and debug logs from MapReduce programs are
not available (<a class="reference external" href="https://issues.cask.co/browse/CDAP-797">CDAP-797</a>).</p>
</li>
<li><p class="first">If the Hive Metastore is restarted while the CDAP Explore Service is running, the
Explore Service remains alive, but becomes unusable. To correct, <a class="reference external" href="http://docs.cask.co/cdap/3.0.0/en/admin-manual/installation/installation.html#starting-services">restart the CDAP Master</a>,
which will restart all services (<a class="reference external" href="https://issues.cask.co/browse/CDAP-1007">CDAP-1007</a>).</p>
</li>
<li><p class="first">User datasets with names starting with <code class="docutils literal notranslate"><span class="pre">&quot;system&quot;</span></code> can potentially cause conflicts
(<a class="reference external" href="https://issues.cask.co/browse/CDAP-1587">CDAP-1587</a>).</p>
</li>
<li><p class="first">Scaling the number of metrics processor instances doesn’t automatically distribute the
processing load to the newer instances of the metrics processor. The CDAP Master needs to be
restarted to effectively distribute the processing across all metrics processor instances
(<a class="reference external" href="https://issues.cask.co/browse/CDAP-1853">CDAP-1853</a>).</p>
</li>
<li><p class="first">Creating a dataset in a non-existent namespace manifests in the RESTful API with an
incorrect error message (<a class="reference external" href="https://issues.cask.co/browse/CDAP-1864">CDAP-1864</a>).</p>
</li>
<li><p class="first">Retrieving multiple metrics—by issuing an HTTP POST request with a JSON list as
the request body that enumerates the name and attributes for each metric—is currently not
supported in the
<a class="reference external" href="http://docs.cask.co/cdap/2.8.0/en/reference-manual/http-restful-api/http-restful-api-v3/metrics.html#query-tips">Metrics HTTP RESTful API v3</a>.
Instead, use the v2 API. It will be supported in a future release.</p>
</li>
<li><p class="first">Typically, datasets are bundled as part of applications. When an application is upgraded and redeployed,
any changes in datasets will not be redeployed. This is because datasets can be shared across applications,
and an incompatible schema change can break other applications that are using the dataset.
A workaround (<a class="reference external" href="https://issues.cask.co/browse/CDAP-1253">CDAP-1253</a>) is to allow <em>unchecked dataset upgrades</em>.
Upgrades cause the dataset metadata, i.e. its specification including properties, to be updated. The dataset
runtime code is also updated. To prevent data loss the existing data and the underlying HBase tables remain as-is.</p>
<p>You can allow <em>unchecked dataset upgrades</em> by setting the configuration property <code class="docutils literal notranslate"><span class="pre">dataset.unchecked.upgrade</span></code>
to <code class="docutils literal notranslate"><span class="pre">true</span></code> in <code class="docutils literal notranslate"><span class="pre">cdap-site.xml</span></code>. This will ensure that datasets are upgraded when the application is redeployed.
When this configuration is set, the recommended process to deploy an upgraded dataset is to first stop
all applications that are using the dataset before deploying the new version of the application.
This lets all containers (flows, services, etc) to pick up the new dataset changes.
When datasets are upgraded using <code class="docutils literal notranslate"><span class="pre">dataset.unchecked.upgrade</span></code>, no schema compatibility checks are performed by the
system. Hence it is very important that the developer verify the backward-compatibility, and makes sure that
other applications that are using the dataset can work with the new changes.</p>
</li>
</ul>
</div>
</div>
<div class="section" id="release-2-8-0">
<h2><a class="reference external" href="http://docs.cask.co/cdap/2.8.0/index.html">Release 2.8.0</a><a class="headerlink" href="#release-2-8-0" title="Permalink to this headline">🔗</a></h2>
<div class="section" id="general">
<h3>General<a class="headerlink" href="#general" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li>The HTTP RESTful API v2 was deprecated, replaced with the
<a class="reference external" href="http://docs.cask.co/cdap/2.8.0/en/reference-manual/http-restful-api/index.html#http-restful-api-v3">namespaced HTTP RESTful API v3</a>.</li>
<li>Added log rotation for CDAP programs running in YARN containers
(<a class="reference external" href="https://issues.cask.co/browse/CDAP-1295">CDAP-1295</a>).</li>
<li>Added the ability to submit to non-default YARN queues to provide
<a class="reference external" href="http://docs.cask.co/cdap/2.8.0/en/admin-manual/operations/resource-guarantees.html#resource-guarantees">resource guarantees</a>
for CDAP Master services, CDAP programs, and Explore Queries
(<a class="reference external" href="https://issues.cask.co/browse/CDAP-1417">CDAP-1417</a>).</li>
<li>Added the ability to <a class="reference external" href="http://docs.cask.co/cdap/2.8.0/en/admin-manual/operations/tx-maintenance.html#tx-maintenance">prune invalid transactions</a>
(<a class="reference external" href="https://issues.cask.co/browse/CDAP-1540">CDAP-1540</a>).</li>
<li>Added the ability to specify <a class="reference external" href="http://docs.cask.co/cdap/2.8.0/en/developers-manual/advanced/application-logback.html#application-logback">custom logback file for CDAP programs</a>
(<a class="reference external" href="https://issues.cask.co/browse/CDAP-1100">CDAP-1100</a>).</li>
<li>System HTTP services now bind to all interfaces (0.0.0.0), rather than 127.0.0.1.</li>
</ul>
</div>
<div class="section" id="id131">
<h3>New Features<a class="headerlink" href="#id131" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li><strong>Command Line Interface (CLI)</strong><ul>
<li>CLI can now directly connect to a CDAP instance of your choice at startup by using
<code class="docutils literal notranslate"><span class="pre">cdap</span> <span class="pre">cli</span> <span class="pre">--uri</span> <span class="pre">&lt;uri&gt;</span></code>.</li>
<li>Support for runtime arguments, which can be listed by running <code class="docutils literal notranslate"><span class="pre">&quot;cdap</span> <span class="pre">cli</span> <span class="pre">--help&quot;</span></code>.</li>
<li>Table rendering can be configured using <code class="docutils literal notranslate"><span class="pre">&quot;cli</span> <span class="pre">render</span> <span class="pre">as</span> <span class="pre">&lt;alt|csv&gt;&quot;</span></code>.
The option <code class="docutils literal notranslate"><span class="pre">&quot;alt&quot;</span></code> is the default, with <code class="docutils literal notranslate"><span class="pre">&quot;csv&quot;</span></code> available for copy &amp; pasting.</li>
<li>Stream statistics can be computed using <code class="docutils literal notranslate"><span class="pre">&quot;get</span> <span class="pre">stream-stats</span> <span class="pre">&lt;stream-id&gt;&quot;</span></code>.</li>
</ul>
</li>
<li><strong>Datasets</strong><ul>
<li>Added an ObjectMappedTable dataset that maps object fields to table columns and that is also explorable.</li>
<li>Added a PartitionedFileSet dataset that allows addressing files by meta data and that is also explorable.</li>
<li>Table datasets now support a multi-get operation for batched reads.</li>
<li>Allow an unchecked dataset upgrade upon application deployment
(<a class="reference external" href="https://issues.cask.co/browse/CDAP-1574">CDAP-1574</a>).</li>
</ul>
</li>
<li><strong>Metrics</strong><ul>
<li>Added new APIs for exploring available metrics, including drilling down into the context of emitted metrics</li>
<li>Added the ability to explore (search) all metrics; previously, this was restricted to custom user metrics</li>
<li>There are new APIs for querying metrics</li>
<li>New capability to break down a metrics time series using the values of one or more tags in its context</li>
</ul>
</li>
<li><strong>Namespaces</strong><ul>
<li>Applications and programs are now managed within namespaces.</li>
<li>Application logs are available within namespaces.</li>
<li>Metrics are now collected and queried within namespaces.</li>
<li>Datasets can now created and managed within namespaces.</li>
<li>Streams are now namespaced for ingestion, fetching, and consuming by programs.</li>
<li>Explore operations are now namespaced.</li>
</ul>
</li>
<li><strong>Preferences</strong><ul>
<li>Users can store preferences (a property map) at the instance, namespace, application,
or program level.</li>
</ul>
</li>
<li><strong>Spark</strong><ul>
<li>Spark now uses a configurer-style API for specifying
(<a class="reference external" href="https://issues.cask.co/browse/CDAP-1134">CDAP-382</a>).</li>
</ul>
</li>
<li><strong>Workflows</strong><ul>
<li>Users can schedule a workflow based on increments of data being ingested into a stream.</li>
<li>Workflows can be stopped.</li>
<li>The execution of a workflow can be forked into parallelized branches.</li>
<li>The runtime arguments for workflow can be scoped.</li>
</ul>
</li>
<li><strong>Workers</strong><ul>
<li>Added <a class="reference external" href="http://docs.cask.co/cdap/2.8.0/en/developers-manual/building-blocks/workers.html#workers">Worker</a>,
a new program type that can be added to CDAP applications, used to run background
processes and (beta feature) can write to streams through the WorkerContext.</li>
</ul>
</li>
<li><strong>Upgrade and Data Migration Tool</strong><ul>
<li>Added an <a class="reference external" href="http://docs.cask.co/cdap/2.8.0/en/admin-manual/installation/installation.html#upgrading-an-existing-version">automated upgrade tool</a>
which supports upgrading from
2.6.x to 2.8.0. (<strong>Note:</strong> Apps need to be both recompiled and re-deployed.)
Upgrade from 2.7.x to 2.8.0 is not currently supported. If you have a use case for it,
please reach out to us at <a class="reference external" href="https://groups.google.com/d/forum/cdap-user">cdap-user&#64;googlegroups.com</a>.</li>
<li>Added a metric migration tool which migrates old metrics to the new 2.8 format.</li>
</ul>
</li>
</ul>
</div>
<div class="section" id="id132">
<h3>Improvement<a class="headerlink" href="#id132" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li>Improved flow performance and scalability with a new distributed queue implementation.</li>
</ul>
</div>
<div class="section" id="id133">
<h3>API Changes<a class="headerlink" href="#id133" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li>The endpoint (<code class="docutils literal notranslate"><span class="pre">GET</span> <span class="pre">&lt;base-url&gt;/data/explore/datasets/&lt;dataset-name&gt;/schema</span></code>) that
retrieved the schema of a dataset’s underlying Hive table has been removed
(<a class="reference external" href="https://issues.cask.co/browse/CDAP-1603">CDAP-1603</a>).</li>
<li>Endpoints have been added to retrieve the CDAP version and the current configurations of
CDAP and HBase (<a class="reference external" href="http://docs.cask.co/cdap/2.8.0/en/reference-manual/http-restful-api/http-restful-api-v3/configuration.html">Configuration HTTP RESTful API</a>).</li>
</ul>
</div>
<div class="section" id="known-issues-280">
<span id="id134"></span><h3>Known Issues<a class="headerlink" href="#known-issues-280" title="Permalink to this headline">🔗</a></h3>
<ul>
<li><p class="first">When running secure Hadoop clusters, metrics and debug logs from MapReduce programs are
not available (<a class="reference external" href="https://issues.cask.co/browse/CDAP-64">CDAP-64</a> and <a class="reference external" href="https://issues.cask.co/browse/CDAP-797">CDAP-797</a>).</p>
</li>
<li><p class="first">If the Hive Metastore is restarted while the CDAP Explore Service is running, the
Explore Service remains alive, but becomes unusable. To correct, <a class="reference external" href="http://docs.cask.co/cdap/2.8.0/en/admin-manual/installation/installation.html#starting-services">restart the CDAP Master</a>,
which will restart all services (<a class="reference external" href="https://issues.cask.co/browse/CDAP-1007">CDAP-1007</a>).</p>
</li>
<li><p class="first">User datasets with names starting with <code class="docutils literal notranslate"><span class="pre">&quot;system&quot;</span></code> can potentially cause conflicts
(<a class="reference external" href="https://issues.cask.co/browse/CDAP-1587">CDAP-1587</a>).</p>
</li>
<li><p class="first">Scaling the number of metrics processor instances doesn’t automatically distribute the
processing load to the newer instances of the metrics processor. The CDAP Master needs to be
restarted to effectively distribute the processing across all metrics processor instances
(<a class="reference external" href="https://issues.cask.co/browse/CDAP-1853">CDAP-1853</a>).</p>
</li>
<li><p class="first">Creating a dataset in a non-existent namespace manifests in the RESTful API with an
incorrect error message (<a class="reference external" href="https://issues.cask.co/browse/CDAP-1864">CDAP-1864</a>).</p>
</li>
<li><p class="first">Retrieving multiple metrics—by issuing an HTTP POST request with a JSON list as
the request body that enumerates the name and attributes for each metric—is currently not
supported in the
<a class="reference external" href="http://docs.cask.co/cdap/2.8.0/en/reference-manual/http-restful-api/http-restful-api-v3/metrics.html#query-tips">Metrics HTTP RESTful API v3</a>.
Instead, use the v2 API. It will be supported in a future release.</p>
</li>
<li><p class="first">Typically, datasets are bundled as part of applications. When an application is upgraded and redeployed,
any changes in datasets will not be redeployed. This is because datasets can be shared across applications,
and an incompatible schema change can break other applications that are using the dataset.
A workaround (<a class="reference external" href="https://issues.cask.co/browse/CDAP-1253">CDAP-1253</a>) is to allow <em>unchecked dataset upgrades</em>.
Upgrades cause the dataset metadata, i.e. its specification including properties, to be updated. The dataset
runtime code is also updated. To prevent data loss the existing data and the underlying HBase tables remain as-is.</p>
<p>You can allow <em>unchecked dataset upgrades</em> by setting the configuration property <code class="docutils literal notranslate"><span class="pre">dataset.unchecked.upgrade</span></code>
to <code class="docutils literal notranslate"><span class="pre">true</span></code> in <code class="docutils literal notranslate"><span class="pre">cdap-site.xml</span></code>. This will ensure that datasets are upgraded when the application is redeployed.
When this configuration is set, the recommended process to deploy an upgraded dataset is to first stop
all applications that are using the dataset before deploying the new version of the application.
This lets all containers (flows, services, etc) to pick up the new dataset changes.
When datasets are upgraded using <code class="docutils literal notranslate"><span class="pre">dataset.unchecked.upgrade</span></code>, no schema compatibility checks are performed by the
system. Hence it is very important that the developer verify the backward-compatibility, and makes sure that
other applications that are using the dataset can work with the new changes.</p>
</li>
<li><p class="first">A race condition resulting in a deadlock can occur when a TwillRunnable container
shutdowns while it still has ZooKeeper events to process. This occasionally surfaces when
running with OpenJDK or JDK7, though not with Oracle JDK6. It is caused by a change in the
<code class="docutils literal notranslate"><span class="pre">ThreadPoolExecutor</span></code> implementation between Oracle JDK6 and OpenJDK/JDK7. Until Apache Twill is
updated in a future version of CDAP, a work-around is to kill the errant process. The YARN
command to list all running applications and their <code class="docutils literal notranslate"><span class="pre">app-id</span></code>s is:</p>
<div class="highlight-java notranslate"><div class="highlight"><pre><span></span><span class="n">yarn</span> <span class="n">application</span> <span class="o">-</span><span class="n">list</span> <span class="o">-</span><span class="n">appStates</span> <span class="n">RUNNING</span>
</pre></div>
</div>
<p>The command to kill a process is:</p>
<div class="highlight-java notranslate"><div class="highlight"><pre><span></span><span class="n">yarn</span> <span class="n">application</span> <span class="o">-</span><span class="n">kill</span> <span class="o">&lt;</span><span class="n">app</span><span class="o">-</span><span class="n">id</span><span class="o">&gt;</span>
</pre></div>
</div>
<p>All versions of CDAP running Apache Twill version 0.4.0 with this configuration can exhibit this
problem (<a class="reference external" href="https://issues.apache.org/jira/browse/TWILL-110">TWILL-110</a>).</p>
</li>
</ul>
</div>
</div>
<div class="section" id="release-2-7-1">
<h2><a class="reference external" href="http://docs.cask.co/cdap/2.7.1/index.html">Release 2.7.1</a><a class="headerlink" href="#release-2-7-1" title="Permalink to this headline">🔗</a></h2>
<div class="section" id="id135">
<h3>API Changes<a class="headerlink" href="#id135" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li>The property <code class="docutils literal notranslate"><span class="pre">security.auth.server.address</span></code> has been deprecated and replaced with
<code class="docutils literal notranslate"><span class="pre">security.auth.server.bind.address</span></code> (<a class="reference external" href="https://issues.cask.co/browse/CDAP-639">CDAP-639</a>,
<a class="reference external" href="https://issues.cask.co/browse/CDAP-1078">CDAP-1078</a>).</li>
</ul>
</div>
<div class="section" id="id136">
<h3>New Features<a class="headerlink" href="#id136" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li><strong>Spark</strong><ul>
<li>Spark now uses a configurer-style API for specifying (<a class="reference external" href="https://issues.cask.co/browse/CDAP-382">CDAP-382</a>).</li>
<li>Spark can now run as a part of a workflow (<a class="reference external" href="https://issues.cask.co/browse/CDAP-465">CDAP-465</a>).</li>
</ul>
</li>
<li><strong>Security</strong><ul>
<li>CDAP Master now obtains and refreshes Kerberos tickets programmatically (<a class="reference external" href="https://issues.cask.co/browse/CDAP-1134">CDAP-1134</a>).</li>
</ul>
</li>
<li><strong>Datasets</strong><ul>
<li>A new, experimental dataset type to support time-partitioned File sets has been added.</li>
<li>Time-partitioned File sets can be queried with Impala on CDH distributions (<a class="reference external" href="https://issues.cask.co/browse/CDAP-926">CDAP-926</a>).</li>
<li>Streams can be made queryable with Impala by deploying an adapter that periodically
converts it into partitions of a time-partitioned File set (<a class="reference external" href="https://issues.cask.co/browse/CDAP-1129">CDAP-1129</a>).</li>
<li>Support for different levels of conflict detection: <code class="docutils literal notranslate"><span class="pre">ROW</span></code>, <code class="docutils literal notranslate"><span class="pre">COLUMN</span></code>, or <code class="docutils literal notranslate"><span class="pre">NONE</span></code> (<a class="reference external" href="https://issues.cask.co/browse/CDAP-1016">CDAP-1016</a>).</li>
<li>Removed support for <code class="docutils literal notranslate"><span class="pre">&#64;DisableTransaction</span></code> (<a class="reference external" href="https://issues.cask.co/browse/CDAP-1279">CDAP-1279</a>).</li>
<li>Support for annotating a stream with a schema (<a class="reference external" href="https://issues.cask.co/browse/CDAP-606">CDAP-606</a>).</li>
<li>A new API for uploading entire files to a stream has been added (<a class="reference external" href="https://issues.cask.co/browse/CDAP-411">CDAP-411</a>).</li>
</ul>
</li>
<li><strong>Workflow</strong><ul>
<li>Workflow now uses a configurer-style API for specifying (<a class="reference external" href="https://issues.cask.co/browse/CDAP-1207">CDAP-1207</a>).</li>
<li>Multiple instances of a workflow can be run concurrently (<a class="reference external" href="https://issues.cask.co/browse/CDAP-513">CDAP-513</a>).</li>
<li>Programs are no longer part of a workflow; instead, they are added in the application
and are referenced by a workflow using their names (<a class="reference external" href="https://issues.cask.co/browse/CDAP-1116">CDAP-1116</a>).</li>
<li>Schedules are now at the application level and properties can be specified for
Schedules; these properties will be passed to the scheduled program as runtime
arguments (<a class="reference external" href="https://issues.cask.co/browse/CDAP-1148">CDAP-1148</a>).</li>
</ul>
</li>
</ul>
</div>
<div class="section" id="known-issues-271">
<span id="id137"></span><h3>Known Issues<a class="headerlink" href="#known-issues-271" title="Permalink to this headline">🔗</a></h3>
<ul>
<li><p class="first">When upgrading an existing CDAP installation to 2.7.1, all metrics are reset.</p>
</li>
<li><p class="first">When running secure Hadoop clusters, metrics and debug logs from MapReduce programs are
not available (<a class="reference external" href="https://issues.cask.co/browse/CDAP-64">CDAP-64</a> and <a class="reference external" href="https://issues.cask.co/browse/CDAP-797">CDAP-797</a>).</p>
</li>
<li><p class="first">When upgrading a cluster from an earlier version of CDAP, warning messages may appear in
the master log indicating that in-transit (emitted, but not yet processed) metrics
system messages could not be decoded (<em>Failed to decode message to MetricsRecord</em>). This
is because of a change in the format of emitted metrics, and can result in a small
amount of metrics data points being lost (<a class="reference external" href="https://issues.cask.co/browse/CDAP-745">CDAP-745</a>).</p>
</li>
<li><p class="first">A race condition resulting in a deadlock can occur when a TwillRunnable container
shutdowns while it still has ZooKeeper events to process. This occasionally surfaces when
running with OpenJDK or JDK7, though not with Oracle JDK6. It is caused by a change in the
<code class="docutils literal notranslate"><span class="pre">ThreadPoolExecutor</span></code> implementation between Oracle JDK6 and OpenJDK/JDK7. Until Apache Twill is
updated in a future version of CDAP, a work-around is to kill the errant process. The YARN
command to list all running applications and their <code class="docutils literal notranslate"><span class="pre">app-id</span></code>s is:</p>
<div class="highlight-java notranslate"><div class="highlight"><pre><span></span><span class="n">yarn</span> <span class="n">application</span> <span class="o">-</span><span class="n">list</span> <span class="o">-</span><span class="n">appStates</span> <span class="n">RUNNING</span>
</pre></div>
</div>
<p>The command to kill a process is:</p>
<div class="highlight-java notranslate"><div class="highlight"><pre><span></span><span class="n">yarn</span> <span class="n">application</span> <span class="o">-</span><span class="n">kill</span> <span class="o">&lt;</span><span class="n">app</span><span class="o">-</span><span class="n">id</span><span class="o">&gt;</span>
</pre></div>
</div>
<p>All versions of CDAP running Apache Twill version 0.4.0 with this configuration can exhibit this
problem (<a class="reference external" href="https://issues.apache.org/jira/browse/TWILL-110">TWILL-110</a>).</p>
</li>
<li><p class="first">Typically, datasets are bundled as part of applications. When an application is upgraded and redeployed,
any changes in datasets will not be redeployed. This is because datasets can be shared across applications,
and an incompatible schema change can break other applications that are using the dataset.
A workaround (<a class="reference external" href="https://issues.cask.co/browse/CDAP-1253">CDAP-1253</a>) is to allow <em>unchecked dataset upgrades</em>.
Upgrades cause the dataset metadata, i.e. its specification including properties, to be updated. The dataset
runtime code is also updated. To prevent data loss the existing data and the underlying HBase tables remain as-is.</p>
<p>You can allow <em>unchecked dataset upgrades</em> by setting the configuration property <code class="docutils literal notranslate"><span class="pre">dataset.unchecked.upgrade</span></code>
to <code class="docutils literal notranslate"><span class="pre">true</span></code> in <code class="docutils literal notranslate"><span class="pre">cdap-site.xml</span></code>. This will ensure that datasets are upgraded when the application is redeployed.
When this configuration is set, the recommended process to deploy an upgraded dataset is to first stop
all applications that are using the dataset before deploying the new version of the application.
This lets all containers (flows, services, etc) to pick up the new dataset changes.
When datasets are upgraded using <code class="docutils literal notranslate"><span class="pre">dataset.unchecked.upgrade</span></code>, no schema compatibility checks are performed by the
system. Hence it is very important that the developer verify the backward-compatibility, and makes sure that
other applications that are using the dataset can work with the new changes.</p>
</li>
</ul>
</div>
</div>
<div class="section" id="release-2-6-1">
<h2><a class="reference external" href="http://docs.cask.co/cdap/2.6.1/index.html">Release 2.6.1</a><a class="headerlink" href="#release-2-6-1" title="Permalink to this headline">🔗</a></h2>
<div class="section" id="cdap-bug-fixes">
<h3>CDAP Bug Fixes<a class="headerlink" href="#cdap-bug-fixes" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li>Allow an <em>unchecked dataset upgrade</em> upon application deployment
(<a class="reference external" href="https://issues.cask.co/browse/CDAP-1253">CDAP-1253</a>).</li>
<li>Update the Hive dataset table when a dataset is updated
(<a class="reference external" href="https://issues.cask.co/browse/CDAP-71">CDAP-71</a>).</li>
<li>Use Hadoop configuration files bundled with the Explore Service
(<a class="reference external" href="https://issues.cask.co/browse/CDAP-1250">CDAP-1250</a>).</li>
</ul>
</div>
<div class="section" id="known-issues-261">
<span id="id138"></span><h3>Known Issues<a class="headerlink" href="#known-issues-261" title="Permalink to this headline">🔗</a></h3>
<ul>
<li><p class="first">When running secure Hadoop clusters, metrics and debug logs from MapReduce programs are
not available (<a class="reference external" href="https://issues.cask.co/browse/CDAP-64">CDAP-64</a> and <a class="reference external" href="https://issues.cask.co/browse/CDAP-797">CDAP-797</a>).</p>
</li>
<li><p class="first">When upgrading a cluster from an earlier version of CDAP, warning messages may appear in
the master log indicating that in-transit (emitted, but not yet processed) metrics
system messages could not be decoded (<em>Failed to decode message to MetricsRecord</em>). This
is because of a change in the format of emitted metrics, and can result in a small
amount of metrics data points being lost (<a class="reference external" href="https://issues.cask.co/browse/CDAP-745">CDAP-745</a>).</p>
</li>
<li><p class="first">A race condition resulting in a deadlock can occur when a TwillRunnable container
shutdowns while it still has ZooKeeper events to process. This occasionally surfaces when
running with OpenJDK or JDK7, though not with Oracle JDK6. It is caused by a change in the
<code class="docutils literal notranslate"><span class="pre">ThreadPoolExecutor</span></code> implementation between Oracle JDK6 and OpenJDK/JDK7. Until Apache Twill is
updated in a future version of CDAP, a work-around is to kill the errant process. The YARN
command to list all running applications and their <code class="docutils literal notranslate"><span class="pre">app-id</span></code>s is:</p>
<div class="highlight-java notranslate"><div class="highlight"><pre><span></span><span class="n">yarn</span> <span class="n">application</span> <span class="o">-</span><span class="n">list</span> <span class="o">-</span><span class="n">appStates</span> <span class="n">RUNNING</span>
</pre></div>
</div>
<p>The command to kill a process is:</p>
<div class="highlight-java notranslate"><div class="highlight"><pre><span></span><span class="n">yarn</span> <span class="n">application</span> <span class="o">-</span><span class="n">kill</span> <span class="o">&lt;</span><span class="n">app</span><span class="o">-</span><span class="n">id</span><span class="o">&gt;</span>
</pre></div>
</div>
<p>All versions of CDAP running Apache Twill version 0.4.0 with this configuration can exhibit this
problem (<a class="reference external" href="https://issues.apache.org/jira/browse/TWILL-110">TWILL-110</a>).</p>
</li>
<li><p class="first">Typically, datasets are bundled as part of applications. When an application is upgraded and redeployed,
any changes in datasets will not be redeployed. This is because datasets can be shared across applications,
and an incompatible schema change can break other applications that are using the dataset.
A workaround (<a class="reference external" href="https://issues.cask.co/browse/CDAP-1253">CDAP-1253</a>) is to allow <em>unchecked dataset upgrades</em>.
Upgrades cause the dataset metadata, i.e. its specification including properties, to be updated. The dataset
runtime code is also updated. To prevent data loss the existing data and the underlying HBase tables remain as-is.</p>
<p>You can allow <em>unchecked dataset upgrades</em> by setting the configuration property <code class="docutils literal notranslate"><span class="pre">dataset.unchecked.upgrade</span></code>
to <code class="docutils literal notranslate"><span class="pre">true</span></code> in <code class="docutils literal notranslate"><span class="pre">cdap-site.xml</span></code>. This will ensure that datasets are upgraded when the application is redeployed.
When this configuration is set, the recommended process to deploy an upgraded dataset is to first stop
all applications that are using the dataset before deploying the new version of the application.
This lets all containers (flows, services, etc) to pick up the new dataset changes.
When datasets are upgraded using <code class="docutils literal notranslate"><span class="pre">dataset.unchecked.upgrade</span></code>, no schema compatibility checks are performed by the
system. Hence it is very important that the developer verify the backward-compatibility, and makes sure that
other applications that are using the dataset can work with the new changes.</p>
</li>
</ul>
</div>
</div>
<div class="section" id="release-2-6-0">
<h2><a class="reference external" href="http://docs.cask.co/cdap/2.6.0/index.html">Release 2.6.0</a><a class="headerlink" href="#release-2-6-0" title="Permalink to this headline">🔗</a></h2>
<div class="section" id="id139">
<h3>API Changes<a class="headerlink" href="#id139" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li>API for specifying services and MapReduce programs has been changed to use a “configurer”
style; this will require modification of user classes implementing either MapReduce
or service as the interfaces have changed (<a class="reference external" href="https://issues.cask.co/browse/CDAP-335">CDAP-335</a>).</li>
</ul>
</div>
<div class="section" id="id140">
<h3>New Features<a class="headerlink" href="#id140" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li><strong>General</strong><ul>
<li>Health checks are now available for CDAP system services
(<a class="reference external" href="https://issues.cask.co/browse/CDAP-663">CDAP-663</a>).</li>
</ul>
</li>
<li><strong>Applications</strong><ul>
<li>Jar deployment now uses a chunked request and writes to a local temp file
(<a class="reference external" href="https://issues.cask.co/browse/CDAP-91">CDAP-91</a>).</li>
</ul>
</li>
<li><strong>MapReduce</strong><ul>
<li>MapReduce programs can now read binary stream data
(<a class="reference external" href="https://issues.cask.co/browse/CDAP-331">CDAP-331</a>).</li>
</ul>
</li>
<li><strong>Datasets</strong><ul>
<li>Added <a class="reference external" href="http://docs.cask.co/cdap/2.6.0/en/developers-manual/building-blocks/datasets/fileset.html#datasets-fileset">FileSet</a>,
a new core dataset type for working with sets of files
(<a class="reference external" href="https://issues.cask.co/browse/CDAP-1">CDAP-1</a>).</li>
</ul>
</li>
<li><strong>Spark</strong><ul>
<li>Spark programs now emit system and custom user metrics
(<a class="reference external" href="https://issues.cask.co/browse/CDAP-346">CDAP-346</a>).</li>
<li>Services can be called from Spark programs and its worker nodes
(<a class="reference external" href="https://issues.cask.co/browse/CDAP-348">CDAP-348</a>).</li>
<li>Spark programs can now read from streams
(<a class="reference external" href="https://issues.cask.co/browse/CDAP-403">CDAP-403</a>).</li>
<li>Added Spark support to the CDAP CLI (Command-line Interface)
(<a class="reference external" href="https://issues.cask.co/browse/CDAP-425">CDAP-425</a>).</li>
<li>Improved speed of Spark unit tests
(<a class="reference external" href="https://issues.cask.co/browse/CDAP-600">CDAP-600</a>).</li>
<li>Spark programs now display system metrics in the CDAP Console
(<a class="reference external" href="https://issues.cask.co/browse/CDAP-652">CDAP-652</a>).</li>
</ul>
</li>
<li><strong>Procedures</strong><ul>
<li>Procedures have been deprecated in favor of services
(<a class="reference external" href="https://issues.cask.co/browse/CDAP-413">CDAP-413</a>).</li>
</ul>
</li>
<li><strong>Services</strong><ul>
<li>Added an HTTP endpoint that returns the endpoints a particular service exposes
(<a class="reference external" href="https://issues.cask.co/browse/CDAP-412">CDAP-412</a>).</li>
<li>Added an HTTP endpoint that lists all services
(<a class="reference external" href="https://issues.cask.co/browse/CDAP-469">CDAP-469</a>).</li>
<li>Default metrics for services have been added to the CDAP Console
(<a class="reference external" href="https://issues.cask.co/browse/CDAP-512">CDAP-512</a>).</li>
<li>The annotations <code class="docutils literal notranslate"><span class="pre">&#64;QueryParam</span></code> and <code class="docutils literal notranslate"><span class="pre">&#64;DefaultValue</span></code> are now supported in custom service handlers
(<a class="reference external" href="https://issues.cask.co/browse/CDAP-664">CDAP-664</a>).</li>
</ul>
</li>
<li><strong>Metrics</strong><ul>
<li>System and user metrics now support gauge metrics
(<a class="reference external" href="https://issues.cask.co/browse/CDAP-484">CDAP-484</a>).</li>
<li>Metrics can be queried using a program’s run-ID
(<a class="reference external" href="https://issues.cask.co/browse/CDAP-620">CDAP-620</a>).</li>
</ul>
</li>
<li><strong>Documentation</strong><ul>
<li>A <a class="reference external" href="http://docs.cask.co/cdap/2.6.0/en/admin-manual/installation/quick-start.html#installation-quick-start">Quick Start Guide</a> has been added to the
<a class="reference external" href="http://docs.cask.co/cdap/2.6.0/en/admin-manual/index.html#admin-index">CDAP 管理手册</a>
(<a class="reference external" href="https://issues.cask.co/browse/CDAP-695">CDAP-695</a>).</li>
</ul>
</li>
</ul>
</div>
<div class="section" id="id141">
<h3>CDAP Bug Fixes<a class="headerlink" href="#id141" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li>Fixed a problem with readless increments not being used when they were enabled in a dataset
(<a class="reference external" href="https://issues.cask.co/browse/CDAP-383">CDAP-383</a>).</li>
<li>Fixed a problem with applications, whose Spark or Scala user classes were not extended
from either <code class="docutils literal notranslate"><span class="pre">JavaSparkProgram</span></code> or <code class="docutils literal notranslate"><span class="pre">ScalaSparkProgram</span></code>, failing with a class loading error
(<a class="reference external" href="https://issues.cask.co/browse/CDAP-599">CDAP-599</a>).</li>
<li>Fixed a problem with the <a class="reference external" href="http://docs.cask.co/cdap/2.6.0/en/admin-manual/installation/installation.html#upgrading-an-existing-version">CDAP upgrade tool</a>
not preserving—for tables with readless increments enabled—the coprocessor
configuration during an upgrade (<a class="reference external" href="https://issues.cask.co/browse/CDAP-1044">CDAP-1044</a>).</li>
<li>Fixed a problem with the readless increment implementation dropping increment cells when
a region flush or compaction occurred (<a class="reference external" href="https://issues.cask.co/browse/CDAP-1062">CDAP-1062</a>).</li>
</ul>
</div>
<div class="section" id="known-issues-260">
<span id="id142"></span><h3>Known Issues<a class="headerlink" href="#known-issues-260" title="Permalink to this headline">🔗</a></h3>
<ul>
<li><p class="first">When running secure Hadoop clusters, metrics and debug logs from MapReduce programs are
not available (<a class="reference external" href="https://issues.cask.co/browse/CDAP-64">CDAP-64</a> and <a class="reference external" href="https://issues.cask.co/browse/CDAP-797">CDAP-797</a>).</p>
</li>
<li><p class="first">When upgrading a cluster from an earlier version of CDAP, warning messages may appear in
the master log indicating that in-transit (emitted, but not yet processed) metrics
system messages could not be decoded (<em>Failed to decode message to MetricsRecord</em>). This
is because of a change in the format of emitted metrics, and can result in a small
amount of metrics data points being lost (<a class="reference external" href="https://issues.cask.co/browse/CDAP-745">CDAP-745</a>).</p>
</li>
<li><p class="first">A race condition resulting in a deadlock can occur when a TwillRunnable container
shutdowns while it still has ZooKeeper events to process. This occasionally surfaces when
running with OpenJDK or JDK7, though not with Oracle JDK6. It is caused by a change in the
<code class="docutils literal notranslate"><span class="pre">ThreadPoolExecutor</span></code> implementation between Oracle JDK6 and OpenJDK/JDK7. Until Apache Twill is
updated in a future version of CDAP, a work-around is to kill the errant process. The YARN
command to list all running applications and their <code class="docutils literal notranslate"><span class="pre">app-id</span></code>s is:</p>
<div class="highlight-java notranslate"><div class="highlight"><pre><span></span><span class="n">yarn</span> <span class="n">application</span> <span class="o">-</span><span class="n">list</span> <span class="o">-</span><span class="n">appStates</span> <span class="n">RUNNING</span>
</pre></div>
</div>
<p>The command to kill a process is:</p>
<div class="highlight-java notranslate"><div class="highlight"><pre><span></span><span class="n">yarn</span> <span class="n">application</span> <span class="o">-</span><span class="n">kill</span> <span class="o">&lt;</span><span class="n">app</span><span class="o">-</span><span class="n">id</span><span class="o">&gt;</span>
</pre></div>
</div>
<p>All versions of CDAP running Apache Twill version 0.4.0 with this configuration can exhibit this
problem (<a class="reference external" href="https://issues.apache.org/jira/browse/TWILL-110">TWILL-110</a>).</p>
</li>
</ul>
</div>
</div>
<div class="section" id="release-2-5-2">
<h2><a class="reference external" href="http://docs.cask.co/cdap/2.5.2/index.html">Release 2.5.2</a><a class="headerlink" href="#release-2-5-2" title="Permalink to this headline">🔗</a></h2>
<div class="section" id="id143">
<h3>CDAP Bug Fixes<a class="headerlink" href="#id143" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li>Fixed a problem with a Coopr-provisioned secure cluster failing to start due to a classpath
issue (<a class="reference external" href="https://issues.cask.co/browse/CDAP-478">CDAP-478</a>).</li>
<li>Fixed a problem with the WISE app zip distribution not packaged correctly; a new version
(0.2.1) has been released (<a class="reference external" href="https://issues.cask.co/browse/CDAP-533">CDAP-533</a>).</li>
<li>Fixed a problem with the examples and tests incorrectly using the ByteBuffer.array
method when reading a stream event (<a class="reference external" href="https://issues.cask.co/browse/CDAP-549">CDAP-549</a>).</li>
<li>Fixed a problem with the Authentication Server so that it can now communicate with an LDAP
instance over SSL (<a class="reference external" href="https://issues.cask.co/browse/CDAP-556">CDAP-556</a>).</li>
<li>Fixed a problem with the program class loader to allow applications to use a different
version of a library than the one that the CDAP platform uses; for example, a different
Kafka library (<a class="reference external" href="https://issues.cask.co/browse/CDAP-559">CDAP-559</a>).</li>
<li>Fixed a problem with CDAP master not obtaining new delegation tokens after running for
<code class="docutils literal notranslate"><span class="pre">hbase.auth.key.update.interval</span></code> milliseconds (<a class="reference external" href="https://issues.cask.co/browse/CDAP-562">CDAP-562</a>).</li>
<li>Fixed a problem with the transaction not being rolled back when a user service handler throws an exception
(<a class="reference external" href="https://issues.cask.co/browse/CDAP-607">CDAP-607</a>).</li>
</ul>
</div>
<div class="section" id="other-changes">
<h3>Other Changes<a class="headerlink" href="#other-changes" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li>Improved the CDAP documentation:<ul>
<li>Re-organized the documentation into three manuals—Developers’ Manual, Administration
Manual, 参考手册—and a set of examples, how-to guides and tutorials;</li>
<li>Documents are now in smaller chapters, with numerous updates and revisions;</li>
<li>Added a link for downloading an archive of the documentation for offline use;</li>
<li>Added links to examples relevant to a particular component;</li>
<li>Added suggested deployment architectures for Distributed CDAP installations;</li>
<li>Added a glossary;</li>
<li>Added navigation aids at the bottom of each page; and</li>
<li>Tested and updated the Standalone CDAP examples and their documentation.</li>
</ul>
</li>
</ul>
</div>
<div class="section" id="id144">
<h3>Known Issues<a class="headerlink" href="#id144" title="Permalink to this headline">🔗</a></h3>
<ul>
<li><p class="first">Currently, applications that include Spark or Scala classes in user classes not extended
from either <code class="docutils literal notranslate"><span class="pre">JavaSparkProgram</span></code> or <code class="docutils literal notranslate"><span class="pre">ScalaSparkProgram</span></code> (depending upon the language)
fail with a class loading error. Spark or Scala classes should not be used outside of the
Spark program. (<a class="reference external" href="https://issues.cask.co/browse/CDAP-599">CDAP-599</a>)</p>
</li>
<li><p class="first">Metrics for MapReduce programs aren’t populated on secure Hadoop clusters</p>
</li>
<li><p class="first">The metric for the number of cores shown in the Resources view of the CDAP Console will be zero
unless YARN has been configured to enable virtual cores</p>
</li>
<li><p class="first">Writing to datasets through Hive is not supported in CDH4.x
(<a class="reference external" href="https://issues.cask.co/browse/CDAP-988">CDAP-988</a>).</p>
</li>
<li><p class="first">A race condition resulting in a deadlock can occur when a TwillRunnable container
shutdowns while it still has ZooKeeper events to process. This occasionally surfaces when
running with OpenJDK or JDK7, though not with Oracle JDK6. It is caused by a change in the
<code class="docutils literal notranslate"><span class="pre">ThreadPoolExecutor</span></code> implementation between Oracle JDK6 and OpenJDK/JDK7. Until Apache Twill is
updated in a future version of CDAP, a work-around is to kill the errant process. The YARN
command to list all running applications and their <code class="docutils literal notranslate"><span class="pre">app-id</span></code>s is:</p>
<div class="highlight-java notranslate"><div class="highlight"><pre><span></span><span class="n">yarn</span> <span class="n">application</span> <span class="o">-</span><span class="n">list</span> <span class="o">-</span><span class="n">appStates</span> <span class="n">RUNNING</span>
</pre></div>
</div>
<p>The command to kill a process is:</p>
<div class="highlight-java notranslate"><div class="highlight"><pre><span></span><span class="n">yarn</span> <span class="n">application</span> <span class="o">-</span><span class="n">kill</span> <span class="o">&lt;</span><span class="n">app</span><span class="o">-</span><span class="n">id</span><span class="o">&gt;</span>
</pre></div>
</div>
<p>All versions of CDAP running Apache Twill version 0.4.0 with this configuration can exhibit this
problem (<a class="reference external" href="https://issues.apache.org/jira/browse/TWILL-110">TWILL-110</a>).</p>
</li>
</ul>
</div>
</div>
<div class="section" id="release-2-5-1">
<h2><a class="reference external" href="http://docs.cask.co/cdap/2.5.1/index.html">Release 2.5.1</a><a class="headerlink" href="#release-2-5-1" title="Permalink to this headline">🔗</a></h2>
<div class="section" id="id145">
<h3>CDAP Bug Fixes<a class="headerlink" href="#id145" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li>Improved the documentation of the CDAP authentication and stream clients, both Java and Python APIs.</li>
<li>Fixed problems with the CDAP Command Line Interface (CLI):<ul>
<li>Did not work in non-interactive mode;</li>
<li>Printed excessive debug log messages;</li>
<li>Relative paths did not work as expected; and</li>
<li>Failed to execute SQL queries.</li>
</ul>
</li>
<li>Removed dependencies on SNAPSHOT artifacts for <em>netty-http</em> and <em>auth-clients</em>.</li>
<li>Corrected an error in the message printed by the startup script <code class="docutils literal notranslate"><span class="pre">cdap</span> <span class="pre">sdk</span></code>.</li>
<li>Resolved a problem with the reading of the properties file by the CDAP Flume Client of CDAP Ingest library
without first checking if authentication was enabled.</li>
</ul>
</div>
<div class="section" id="id146">
<h3>Other Changes<a class="headerlink" href="#id146" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li>The scripts <code class="docutils literal notranslate"><span class="pre">send-query.sh</span></code>, <code class="docutils literal notranslate"><span class="pre">access-token.sh</span></code> and <code class="docutils literal notranslate"><span class="pre">access-token.bat</span></code> has been replaced by the
<a class="reference external" href="http://docs.cask.co/cdap/2.5.1/en/api.html#command-line-interface">CDAP Command Line Interface</a>, <code class="docutils literal notranslate"><span class="pre">cdap</span> <span class="pre">cli</span></code>.</li>
<li>The CDAP Command Line Interface now uses and saves access tokens when connecting to a secure CDAP instance.</li>
<li>The CDAP Java Stream Client now allows empty String events to be sent.</li>
<li>The CDAP Python Authentication Client’s <code class="docutils literal notranslate"><span class="pre">configure()</span></code> method now takes a dictionary rather than a filepath.</li>
</ul>
</div>
<div class="section" id="id147">
<h3>Known Issues<a class="headerlink" href="#id147" title="Permalink to this headline">🔗</a></h3>
<ul>
<li><p class="first">Metrics for MapReduce programs aren’t populated on secure Hadoop clusters</p>
</li>
<li><p class="first">The metric for the number of cores shown in the Resources view of the CDAP Console will be zero
unless YARN has been configured to enable virtual cores</p>
</li>
<li><p class="first">A race condition resulting in a deadlock can occur when a TwillRunnable container
shutdowns while it still has ZooKeeper events to process. This occasionally surfaces when
running with OpenJDK or JDK7, though not with Oracle JDK6. It is caused by a change in the
<code class="docutils literal notranslate"><span class="pre">ThreadPoolExecutor</span></code> implementation between Oracle JDK6 and OpenJDK/JDK7. Until Apache Twill is
updated in a future version of CDAP, a work-around is to kill the errant process. The YARN
command to list all running applications and their <code class="docutils literal notranslate"><span class="pre">app-id</span></code>s is:</p>
<div class="highlight-java notranslate"><div class="highlight"><pre><span></span><span class="n">yarn</span> <span class="n">application</span> <span class="o">-</span><span class="n">list</span> <span class="o">-</span><span class="n">appStates</span> <span class="n">RUNNING</span>
</pre></div>
</div>
<p>The command to kill a process is:</p>
<div class="highlight-java notranslate"><div class="highlight"><pre><span></span><span class="n">yarn</span> <span class="n">application</span> <span class="o">-</span><span class="n">kill</span> <span class="o">&lt;</span><span class="n">app</span><span class="o">-</span><span class="n">id</span><span class="o">&gt;</span>
</pre></div>
</div>
<p>All versions of CDAP running Apache Twill version 0.4.0 with this configuration can exhibit this
problem (<a class="reference external" href="https://issues.apache.org/jira/browse/TWILL-110">TWILL-110</a>).</p>
</li>
</ul>
</div>
</div>
<div class="section" id="release-2-5-0">
<h2><a class="reference external" href="http://docs.cask.co/cdap/2.5.0/index.html">Release 2.5.0</a><a class="headerlink" href="#release-2-5-0" title="Permalink to this headline">🔗</a></h2>
<div class="section" id="id148">
<h3>New Features<a class="headerlink" href="#id148" title="Permalink to this headline">🔗</a></h3>
<div class="section" id="ad-hoc-querying">
<h4>Ad-hoc querying<a class="headerlink" href="#ad-hoc-querying" title="Permalink to this headline">🔗</a></h4>
<ul class="simple">
<li>Capability to write to datasets using SQL</li>
<li>Added a CDAP JDBC driver allowing connections from Java applications and third-party business intelligence tools</li>
<li>Ability to perform ad-hoc queries from the CDAP Console:<ul>
<li>Execute a SQL query from the Console</li>
<li>View list of active, completed queries</li>
<li>Download query results</li>
</ul>
</li>
</ul>
</div>
<div class="section" id="datasets">
<h4>Datasets<a class="headerlink" href="#datasets" title="Permalink to this headline">🔗</a></h4>
<ul class="simple">
<li>Datasets can be tested with TestBase outside of the context of an application</li>
<li>CDAP now checks datasets for compatibility in a verification stage</li>
<li>The Transaction engine uses server-side filtering for efficient transactional reads</li>
<li>Dataset specifications can now be dynamically reconfigured through the use of RESTful endpoints</li>
<li>The Bundle jar format is now used for dataset libs</li>
<li>Increments on datasets are now read-less</li>
</ul>
</div>
<div class="section" id="services">
<h4>Services<a class="headerlink" href="#services" title="Permalink to this headline">🔗</a></h4>
<ul class="simple">
<li>Added simplified APIs for using services from other programs such as MapReduce, flows and Procedures</li>
<li>Added an API for creating services and handlers that can use datasets transactionally</li>
<li>Added a RESTful API to make requests to a service via the Router</li>
</ul>
</div>
<div class="section" id="security">
<h4>Security<a class="headerlink" href="#security" title="Permalink to this headline">🔗</a></h4>
<ul class="simple">
<li>Added authorization logging</li>
<li>Added Kerberos authentication to ZooKeeper secret keys</li>
<li>Added support for SSL</li>
</ul>
</div>
<div class="section" id="spark-integration">
<h4>Spark Integration<a class="headerlink" href="#spark-integration" title="Permalink to this headline">🔗</a></h4>
<ul class="simple">
<li>Supports running Spark programs as a part of CDAP applications in Standalone mode</li>
<li>Supports running Spark programs written with Spark versions 1.0.1 or 1.1.0</li>
<li>Supports Spark’s <em>MLib</em> and <em>GraphX</em> modules</li>
<li>Includes three examples demonstrating CDAP Spark programs</li>
<li>Adds display of Spark program logs and history in the CDAP Console</li>
</ul>
</div>
<div class="section" id="streams">
<h4>Streams<a class="headerlink" href="#streams" title="Permalink to this headline">🔗</a></h4>
<ul class="simple">
<li>Added a collection of applications, tools and APIs specifically for the ETL (Extract, Transform and Loading) of data</li>
<li>Added support for asynchronously writing to streams</li>
</ul>
</div>
<div class="section" id="clients">
<h4>Clients<a class="headerlink" href="#clients" title="Permalink to this headline">🔗</a></h4>
<ul class="simple">
<li>Added a Command Line Interface</li>
<li>Added a Java Client Interface</li>
</ul>
</div>
</div>
<div class="section" id="major-cdap-bug-fixes">
<h3>Major CDAP Bug Fixes<a class="headerlink" href="#major-cdap-bug-fixes" title="Permalink to this headline">🔗</a></h3>
<ul class="simple">
<li>Fixed a problem with a HADOOP_HOME exception stacktrace when unit-testing an application</li>
<li>Fixed an issue with Hive creating directories in /tmp in the Standalone and unit-test frameworks</li>
<li>Fixed a problem with type inconsistency of service API calls, where numbers were showing up as strings</li>
<li>Fixed an issue with the premature expiration of long-term Authentication Tokens</li>
<li>Fixed an issue with the dataset size metric showing data operations size instead of resource usage</li>
</ul>
</div>
<div class="section" id="known-issues-250">
<span id="id149"></span><h3>Known Issues<a class="headerlink" href="#known-issues-250" title="Permalink to this headline">🔗</a></h3>
<ul>
<li><p class="first">Metrics for MapReduce programs aren’t populated on secure Hadoop clusters</p>
</li>
<li><p class="first">The metric for the number of cores shown in the Resources view of the CDAP Console will be zero
unless YARN has been configured to enable virtual cores</p>
</li>
<li><p class="first">A race condition resulting in a deadlock can occur when a TwillRunnable container
shutdowns while it still has ZooKeeper events to process. This occasionally surfaces when
running with OpenJDK or JDK7, though not with Oracle JDK6. It is caused by a change in the
<code class="docutils literal notranslate"><span class="pre">ThreadPoolExecutor</span></code> implementation between Oracle JDK6 and OpenJDK/JDK7. Until Apache Twill is
updated in a future version of CDAP, a work-around is to kill the errant process. The YARN
command to list all running applications and their <code class="docutils literal notranslate"><span class="pre">app-id</span></code>s is:</p>
<div class="highlight-java notranslate"><div class="highlight"><pre><span></span><span class="n">yarn</span> <span class="n">application</span> <span class="o">-</span><span class="n">list</span> <span class="o">-</span><span class="n">appStates</span> <span class="n">RUNNING</span>
</pre></div>
</div>
<p>The command to kill a process is:</p>
<div class="highlight-java notranslate"><div class="highlight"><pre><span></span><span class="n">yarn</span> <span class="n">application</span> <span class="o">-</span><span class="n">kill</span> <span class="o">&lt;</span><span class="n">app</span><span class="o">-</span><span class="n">id</span><span class="o">&gt;</span>
</pre></div>
</div>
<p>All versions of CDAP running Apache Twill version 0.4.0 with this configuration can exhibit this
problem (<a class="reference external" href="https://issues.apache.org/jira/browse/TWILL-110">TWILL-110</a>).</p>
</li>
</ul>
</div>
</div>
</div>

</div>
    <div class="col-md-2">
      <div id="right-sidebar" class="bs-sidenav scrollable-y" role="complementary">
        <div id="localtoc-scrollspy">
        </div>
      </div>
    </div></div>
</div>
<!-- block main content end -->
<!-- block footer -->
<footer class="footer">
      <div class="container">
        <div class="row">
          <div class="col-md-2 footer-left"><a title="Javadocs" href="javadocs/index.html" />Previous</a></div>
          <div class="col-md-8 footer-center"><a class="footer-tab-link" href="table-of-contents/../../reference-manual/licenses/index.html">Copyright</a> &copy; 2014-2020 Cask Data, Inc.&bull; <a class="footer-tab-link" href="//docs.cask.co/cdap/6.1.1/cdap-docs-6.1.1-web.zip" rel="nofollow">Download</a> an archive or
<a class="footer-tab-link" href="//docs.cask.co/cdap">switch the version</a> of the documentation
          </div>
          <div class="col-md-2 footer-right"><a title="Licenses, and Dependencies" href="licenses/index.html" />Next</a></div>
        </div>
      </div>
    </footer>
<!-- block footer end -->
<script type="text/javascript" src="_static/bootstrap-3.3.6/js/bootstrap.min.js"></script><script type="text/javascript" src="_static/js/bootstrap-sphinx.js"></script><script type="text/javascript" src="_static/js/abixTreeList-2.js"></script><script type="text/javascript" src="_static/js/cdap-dynamicscrollspy-4.js"></script><script type="text/javascript" src="_static/js/cdap-version-menu.js"></script><script type="text/javascript" src="_static/js/copy-to-clipboard.js"></script><script type="text/javascript" src="_static/js/jquery.mousewheel.min.js"></script><script type="text/javascript" src="_static/js/jquery.mCustomScrollbar.js"></script><script type="text/javascript" src="_static/js/js.cookie.js"></script><script type="text/javascript" src="_static/js/tabbed-parsed-literal-0.2.js"></script><script type="text/javascript" src="_static/js/cdap-onload-javascript.js"></script><script type="text/javascript" src="_static/js/cdap-version-menu.js"></script>
    <script src="https://cdap.gitee.io/docs/cdap/json-versions.js"/></script>
  </body>
</html>